提交

src/database/models/__init__.py

@@ -0,0 +1,17 @@
+from src.database.models.crawl_task import CrawlTask
+from src.database.models.task_search import TaskSearch
+from src.database.models.task_page import TaskPageAnalysis
+from src.database.models.task_next import TaskNextPage
+from src.database.models.task_detail import TaskDetailAnalysis
+from src.database.models.task_crawl import TaskCrawl
+from src.database.models.job_data import JobData
+
+__all__ = [
+    "CrawlTask",
+    "TaskSearch",
+    "TaskPageAnalysis",
+    "TaskNextPage",
+    "TaskDetailAnalysis",
+    "TaskCrawl",
+    "JobData",
+]

src/database/models/crawl_task.py

@@ -0,0 +1,27 @@
+"""爬虫任务主表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class CrawlTask(Base):
+    """爬虫任务主表 app_crawl_task"""
+    __tablename__ = "app_crawl_task"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    company_name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
+    
+    # 配置阶段
+    config_step: Mapped[int] = mapped_column(Integer, default=1)
+    config_status: Mapped[str] = mapped_column(String(32), default="pending")
+    
+    # 爬取阶段
+    crawl_status: Mapped[str | None] = mapped_column(String(32), default=None)
+    total_crawl_times: Mapped[int] = mapped_column(Integer, default=0)
+    last_crawl_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    
+    # 时间戳
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/job_data.py

@@ -0,0 +1,41 @@
+"""Job data table."""
+from datetime import datetime
+
+from sqlalchemy import BigInteger, String, DateTime, Text, SmallInteger
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class JobData(Base):
+    """岗位数据表 app_job_data"""
+
+    __tablename__ = "app_job_data"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    task_crawl_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+
+    job_title: Mapped[str | None] = mapped_column(String(255), default=None, index=True)
+    salary: Mapped[str | None] = mapped_column(String(128), default=None)
+    location: Mapped[str | None] = mapped_column(String(2048), default=None)
+    company: Mapped[str | None] = mapped_column(String(255), default=None)
+    experience: Mapped[str | None] = mapped_column(String(64), default=None)
+    education: Mapped[str | None] = mapped_column(String(64), default=None)
+    description: Mapped[str | None] = mapped_column(Text, default=None)
+    detail_url: Mapped[str | None] = mapped_column(String(1024), default=None)
+
+    company_name_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    job_unique_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+    recruit_category: Mapped[int] = mapped_column(SmallInteger, default=0, index=True)
+
+    sources: Mapped[int] = mapped_column(SmallInteger, default=0)
+    is_independent_url: Mapped[int] = mapped_column(SmallInteger, default=1)
+    is_valid: Mapped[int] = mapped_column(SmallInteger, default=1, index=True)
+
+    expire_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    check_status: Mapped[str] = mapped_column(String(32), default="pending")
+    last_check_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/task_crawl.py

@@ -0,0 +1,30 @@
+"""Step5-数据爬取任务表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime, Text, JSON
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class TaskCrawl(Base):
+    """数据爬取任务表 app_task_crawl"""
+    __tablename__ = "app_task_crawl"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    crawl_task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+    
+    status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
+    retry_count: Mapped[int] = mapped_column(Integer, default=0)
+    max_retry: Mapped[int] = mapped_column(Integer, default=1)
+    
+    # 输入
+    input_config: Mapped[dict] = mapped_column(JSON, nullable=False)
+    
+    # 输出
+    crawled_count: Mapped[int | None] = mapped_column(Integer, default=None)
+    
+    error_message: Mapped[str | None] = mapped_column(Text, default=None)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/task_detail.py

@@ -0,0 +1,33 @@
+"""Step4-详情页分析任务表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime, Text, JSON
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class TaskDetailAnalysis(Base):
+    """详情页分析任务表 app_task_detail_analysis"""
+    __tablename__ = "app_task_detail_analysis"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    crawl_task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+    
+    status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
+    retry_count: Mapped[int] = mapped_column(Integer, default=0)
+    max_retry: Mapped[int] = mapped_column(Integer, default=1)
+    
+    # 输入
+    input_url: Mapped[str] = mapped_column(String(1024), nullable=False)
+    input_job_selector: Mapped[str] = mapped_column(String(512), nullable=False)
+    input_change_type: Mapped[str] = mapped_column(String(32), nullable=False)
+    
+    # 输出
+    output_detail_selector: Mapped[str | None] = mapped_column(String(512), default=None)
+    output_fields: Mapped[dict | None] = mapped_column(JSON, default=None)
+    
+    error_message: Mapped[str | None] = mapped_column(Text, default=None)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/task_next.py

@@ -0,0 +1,32 @@
+"""Step3-分页分析任务表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime, Text, SmallInteger
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class TaskNextPage(Base):
+    """分页分析任务表 app_task_next_page"""
+    __tablename__ = "app_task_next_page"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    crawl_task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+    
+    status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
+    retry_count: Mapped[int] = mapped_column(Integer, default=0)
+    max_retry: Mapped[int] = mapped_column(Integer, default=1)
+    
+    # 输入
+    input_url: Mapped[str] = mapped_column(String(1024), nullable=False)
+    
+    # 输出
+    output_selector: Mapped[str | None] = mapped_column(String(512), default=None)
+    output_change_type: Mapped[str | None] = mapped_column(String(32), default=None)
+    has_pagination: Mapped[int | None] = mapped_column(SmallInteger, default=None)
+    
+    error_message: Mapped[str | None] = mapped_column(Text, default=None)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/task_page.py

@@ -0,0 +1,32 @@
+"""Step2-岗位列表分析任务表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime, Text
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class TaskPageAnalysis(Base):
+    """岗位列表分析任务表 app_task_page_analysis"""
+    __tablename__ = "app_task_page_analysis"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    crawl_task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+    
+    status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
+    retry_count: Mapped[int] = mapped_column(Integer, default=0)
+    max_retry: Mapped[int] = mapped_column(Integer, default=1)
+    
+    # 输入
+    input_url: Mapped[str] = mapped_column(String(1024), nullable=False)
+    
+    # 输出
+    output_selector: Mapped[str | None] = mapped_column(String(512), default=None)
+    output_change_type: Mapped[str | None] = mapped_column(String(32), default=None)
+    output_item_count: Mapped[int | None] = mapped_column(Integer, default=None)
+    
+    error_message: Mapped[str | None] = mapped_column(Text, default=None)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)

src/database/models/task_search.py

@@ -0,0 +1,29 @@
+"""Step1-搜索招聘页面任务表"""
+from datetime import datetime
+from sqlalchemy import BigInteger, String, Integer, DateTime, Text
+from sqlalchemy.orm import Mapped, mapped_column
+
+from src.database.base import Base
+
+
+class TaskSearch(Base):
+    """搜索招聘页面任务表 app_task_search"""
+    __tablename__ = "app_task_search"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
+    crawl_task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
+    
+    status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
+    retry_count: Mapped[int] = mapped_column(Integer, default=0)
+    max_retry: Mapped[int] = mapped_column(Integer, default=1)
+
+    # 输入输出
+    input_company_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    input_url: Mapped[str | None] = mapped_column(String(1024), default=None, comment="已知招聘地址(表格导入)")
+    output_url: Mapped[str | None] = mapped_column(String(512), default=None)
+    
+    error_message: Mapped[str | None] = mapped_column(Text, default=None)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.now, onupdate=datetime.now)