Files
MemRelay/backend/migrations/versions/20260804_0005_curation_git.py

478 lines
24 KiB
Python

"""Add AI curation and optional memory Git data model."""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "20260804_0005"
down_revision: str | Sequence[str] | None = "20260802_0004"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"usage_profiles",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(100), nullable=False, unique=True),
sa.Column("description", sa.Text()),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("is_shared", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_usage_profiles_enabled", "usage_profiles", ["enabled"])
with op.batch_alter_table("projects") as batch:
batch.add_column(
sa.Column(
"workspace_type",
sa.String(32),
nullable=False,
server_default="general",
)
)
batch.create_index("ix_projects_workspace_type", ["workspace_type"])
op.execute(
"UPDATE projects SET workspace_type = 'development' "
"WHERE git_remote IS NOT NULL AND TRIM(git_remote) <> ''"
)
with op.batch_alter_table("mcp_tokens") as batch:
batch.add_column(sa.Column("usage_profile_id", sa.String(36)))
batch.create_index("ix_mcp_tokens_usage_profile_id", ["usage_profile_id"])
batch.create_foreign_key(
"fk_mcp_tokens_usage_profile",
"usage_profiles",
["usage_profile_id"],
["id"],
ondelete="SET NULL",
)
with op.batch_alter_table("memory_references") as batch:
batch.add_column(sa.Column("usage_profile_id", sa.String(36)))
batch.create_index("ix_memory_references_usage_profile_id", ["usage_profile_id"])
batch.create_foreign_key(
"fk_memory_references_usage_profile",
"usage_profiles",
["usage_profile_id"],
["id"],
ondelete="SET NULL",
)
op.create_table(
"curation_settings",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("timezone", sa.String(100), nullable=False, server_default="Asia/Shanghai"),
sa.Column("context_input_tokens", sa.Integer(), nullable=False, server_default="32000"),
sa.Column("context_output_tokens", sa.Integer(), nullable=False, server_default="8000"),
sa.Column("task_max_calls", sa.Integer(), nullable=False, server_default="20"),
sa.Column("task_max_tokens", sa.Integer(), nullable=False, server_default="200000"),
sa.Column("batch_chars", sa.Integer(), nullable=False, server_default="80000"),
sa.Column("max_merge_levels", sa.Integer(), nullable=False, server_default="4"),
sa.Column("max_concurrency", sa.Integer(), nullable=False, server_default="1"),
sa.Column("source_freshness_hours", sa.Integer(), nullable=False, server_default="24"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_curation_settings_enabled", "curation_settings", ["enabled"])
op.create_table(
"curation_model_configs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("revision", sa.Integer(), nullable=False, unique=True),
sa.Column("name", sa.String(100), nullable=False),
sa.Column("base_url", sa.Text(), nullable=False),
sa.Column("encrypted_token", sa.Text()),
sa.Column("text_model", sa.String(300), nullable=False),
sa.Column("vision_model", sa.String(300)),
sa.Column("protocol", sa.String(32), nullable=False, server_default="auto"),
sa.Column("effective_protocol", sa.String(32)),
sa.Column("stream", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("timeout_seconds", sa.Integer(), nullable=False, server_default="120"),
sa.Column("probe_interval_seconds", sa.Integer(), nullable=False, server_default="300"),
sa.Column("management_url", sa.Text()),
sa.Column("capability_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("active", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_curation_model_configs_revision", "curation_model_configs", ["revision"])
op.create_index("ix_curation_model_configs_active", "curation_model_configs", ["active"])
op.create_table(
"curation_dependency_state",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("status", sa.String(32), nullable=False, server_default="unconfigured"),
sa.Column("capability_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("last_check_at", sa.DateTime(timezone=True)),
sa.Column("last_success_at", sa.DateTime(timezone=True)),
sa.Column("last_failure_at", sa.DateTime(timezone=True)),
sa.Column("http_status", sa.Integer()),
sa.Column("error_code", sa.String(100)),
sa.Column("error_message", sa.Text()),
sa.Column("next_probe_at", sa.DateTime(timezone=True)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_curation_dependency_state_status", "curation_dependency_state", ["status"])
op.create_index(
"ix_curation_dependency_state_next_probe_at",
"curation_dependency_state",
["next_probe_at"],
)
op.create_table(
"curation_schedules",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("trigger_type", sa.String(40), nullable=False),
sa.Column("scope", sa.String(16), nullable=False, server_default="project"),
sa.Column("project_id", sa.String(36), sa.ForeignKey("projects.id", ondelete="CASCADE")),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("inheritance", sa.String(20), nullable=False, server_default="instance"),
sa.Column("timezone", sa.String(100)),
sa.Column("recurrence_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("missed_policy", sa.String(20), nullable=False, server_default="run_once"),
sa.Column("last_scheduled_at", sa.DateTime(timezone=True)),
sa.Column("last_triggered_at", sa.DateTime(timezone=True)),
sa.Column("next_run_at", sa.DateTime(timezone=True)),
sa.Column("status", sa.String(32), nullable=False, server_default="active"),
sa.Column("last_error", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_curation_schedules_trigger_type", "curation_schedules", ["trigger_type"])
op.create_index("ix_curation_schedules_project_id", "curation_schedules", ["project_id"])
op.create_index("ix_curation_schedules_enabled", "curation_schedules", ["enabled"])
op.create_index("ix_curation_schedules_next_run_at", "curation_schedules", ["next_run_at"])
op.create_table(
"curation_change_log",
sa.Column("sequence", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("scope", sa.String(16), nullable=False),
sa.Column("project_id", sa.String(36), sa.ForeignKey("projects.id", ondelete="SET NULL")),
sa.Column("source_type", sa.String(40), nullable=False),
sa.Column("source_id", sa.String(100), nullable=False),
sa.Column("change_type", sa.String(20), nullable=False),
sa.Column("revision", sa.String(100)),
sa.Column("content_hash", sa.String(64)),
sa.Column("target_hint", sa.String(200)),
sa.Column("summary", sa.Text()),
sa.Column(
"usage_profile_id",
sa.String(36),
sa.ForeignKey("usage_profiles.id", ondelete="SET NULL"),
),
sa.Column("origin", sa.String(20), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
)
for column in (
"scope",
"project_id",
"source_type",
"source_id",
"usage_profile_id",
"origin",
"observed_at",
):
op.create_index(f"ix_curation_change_log_{column}", "curation_change_log", [column])
op.create_table(
"curation_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scope", sa.String(16), nullable=False),
sa.Column("project_id", sa.String(36), sa.ForeignKey("projects.id", ondelete="SET NULL")),
sa.Column(
"usage_profile_id",
sa.String(36),
sa.ForeignKey("usage_profiles.id", ondelete="SET NULL"),
),
sa.Column("mode", sa.String(20), nullable=False, server_default="incremental"),
sa.Column("trigger", sa.String(40), nullable=False),
sa.Column("source_strategy", sa.String(20), nullable=False, server_default="auto"),
sa.Column("target_documents_json", sa.Text(), nullable=False, server_default="[]"),
sa.Column("coalesce_key", sa.String(200)),
sa.Column("slot", sa.String(240), unique=True),
sa.Column("status", sa.String(32), nullable=False, server_default="queued"),
sa.Column("last_success_cursor", sa.Integer(), nullable=False, server_default="0"),
sa.Column("latest_observed_cursor", sa.Integer(), nullable=False, server_default="0"),
sa.Column("merged_trigger_count", sa.Integer(), nullable=False, server_default="1"),
sa.Column("trigger_summary_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column(
"model_config_id",
sa.String(36),
sa.ForeignKey("curation_model_configs.id", ondelete="SET NULL"),
),
sa.Column("prompt_version", sa.String(100)),
sa.Column("schema_version", sa.String(100)),
sa.Column("lease_owner", sa.String(100)),
sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
sa.Column("retries", sa.Integer(), nullable=False, server_default="0"),
sa.Column("next_retry_at", sa.DateTime(timezone=True)),
sa.Column("error_code", sa.String(100)),
sa.Column("error_message", sa.Text()),
sa.Column("stats_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("started_at", sa.DateTime(timezone=True)),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
for column in (
"scope",
"project_id",
"usage_profile_id",
"trigger",
"coalesce_key",
"status",
"lease_owner",
"lease_expires_at",
"next_retry_at",
):
op.create_index(f"ix_curation_jobs_{column}", "curation_jobs", [column])
op.create_table(
"curation_attempts",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"job_id",
sa.String(36),
sa.ForeignKey("curation_jobs.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("attempt_number", sa.Integer(), nullable=False),
sa.Column(
"model_config_id",
sa.String(36),
sa.ForeignKey("curation_model_configs.id", ondelete="SET NULL"),
),
sa.Column("effective_protocol", sa.String(32)),
sa.Column("model", sa.String(300)),
sa.Column("prompt_version", sa.String(100), nullable=False, server_default="1"),
sa.Column("schema_version", sa.String(100), nullable=False, server_default="1"),
sa.Column("budget_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("status", sa.String(32), nullable=False, server_default="running"),
sa.Column("error_code", sa.String(100)),
sa.Column("error_message", sa.Text()),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.UniqueConstraint("job_id", "attempt_number", name="uq_curation_attempt"),
)
op.create_index("ix_curation_attempts_job_id", "curation_attempts", ["job_id"])
op.create_table(
"curated_documents",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scope", sa.String(16), nullable=False),
sa.Column("project_id", sa.String(36), sa.ForeignKey("projects.id", ondelete="CASCADE")),
sa.Column(
"usage_profile_id",
sa.String(36),
sa.ForeignKey("usage_profiles.id", ondelete="SET NULL"),
),
sa.Column("document_type", sa.String(100), nullable=False),
sa.Column("title", sa.String(300), nullable=False),
sa.Column("path", sa.Text(), nullable=False, unique=True),
sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
sa.Column(
"latest_job_id", sa.String(36), sa.ForeignKey("curation_jobs.id", ondelete="SET NULL")
),
sa.Column("source_hash", sa.String(64), nullable=False),
sa.Column("source_cursor", sa.Integer(), nullable=False, server_default="0"),
sa.Column("model", sa.String(300)),
sa.Column("prompt_version", sa.String(100), nullable=False, server_default="1"),
sa.Column("status", sa.String(32), nullable=False, server_default="active"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"scope",
"project_id",
"usage_profile_id",
"document_type",
name="uq_curated_document",
),
)
for column in ("scope", "project_id", "usage_profile_id", "document_type", "status"):
op.create_index(f"ix_curated_documents_{column}", "curated_documents", [column])
op.create_table(
"curated_document_revisions",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"document_id",
sa.String(36),
sa.ForeignKey("curated_documents.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("storage_path", sa.Text(), nullable=False, unique=True),
sa.Column("job_id", sa.String(36), sa.ForeignKey("curation_jobs.id", ondelete="SET NULL")),
sa.Column("model", sa.String(300)),
sa.Column("source_hash", sa.String(64), nullable=False),
sa.Column("change_summary", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("document_id", "revision", name="uq_curated_document_revision"),
)
op.create_index(
"ix_curated_document_revisions_document_id",
"curated_document_revisions",
["document_id"],
)
op.create_index(
"ix_curated_document_revisions_job_id", "curated_document_revisions", ["job_id"]
)
op.create_table(
"curation_sources",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"job_id",
sa.String(36),
sa.ForeignKey("curation_jobs.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("source_type", sa.String(40), nullable=False),
sa.Column("source_id", sa.String(100), nullable=False),
sa.Column("source_revision", sa.String(100)),
sa.Column("content_hash", sa.String(64)),
sa.Column("origin", sa.String(20), nullable=False),
sa.Column("sent_to_model", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("disposition", sa.String(20), nullable=False, server_default="processed"),
sa.Column("reason", sa.Text()),
sa.UniqueConstraint("job_id", "source_type", "source_id", name="uq_curation_source"),
)
op.create_index("ix_curation_sources_job_id", "curation_sources", ["job_id"])
op.create_table(
"curation_model_calls",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"job_id",
sa.String(36),
sa.ForeignKey("curation_jobs.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"attempt_id", sa.String(36), sa.ForeignKey("curation_attempts.id", ondelete="SET NULL")
),
sa.Column("purpose", sa.String(40), nullable=False),
sa.Column("protocol", sa.String(32), nullable=False),
sa.Column("stream", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("requested_model", sa.String(300), nullable=False),
sa.Column("response_model", sa.String(300)),
sa.Column("request_id", sa.String(300)),
sa.Column("prompt_version", sa.String(100), nullable=False),
sa.Column("input_tokens", sa.Integer()),
sa.Column("output_tokens", sa.Integer()),
sa.Column("latency_ms", sa.Integer()),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("error_code", sa.String(100)),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True)),
)
op.create_index("ix_curation_model_calls_job_id", "curation_model_calls", ["job_id"])
op.create_index("ix_curation_model_calls_attempt_id", "curation_model_calls", ["attempt_id"])
op.create_table(
"git_connections",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("name", sa.String(100), nullable=False),
sa.Column("mode", sa.String(20), nullable=False, server_default="single"),
sa.Column("remote_url", sa.Text()),
sa.Column("username", sa.String(500)),
sa.Column("transport", sa.String(20), nullable=False, server_default="https"),
sa.Column("credential_storage", sa.String(20), nullable=False, server_default="local"),
sa.Column("encrypted_credential", sa.Text()),
sa.Column("vault_item_id", sa.String(100)),
sa.Column("default_branch", sa.String(200), nullable=False, server_default="main"),
sa.Column("author_name", sa.String(200), nullable=False, server_default="MemRelay"),
sa.Column(
"author_email", sa.String(320), nullable=False, server_default="memrelay@localhost"
),
sa.Column("allow_write_test", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("allow_push_to_create", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("capability_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("last_error", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_git_connections_enabled", "git_connections", ["enabled"])
op.create_table(
"memory_repositories",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column(
"connection_id", sa.String(36), sa.ForeignKey("git_connections.id", ondelete="SET NULL")
),
sa.Column("mode", sa.String(20), nullable=False),
sa.Column("scope", sa.String(16), nullable=False),
sa.Column("project_id", sa.String(36), sa.ForeignKey("projects.id", ondelete="SET NULL")),
sa.Column("local_path", sa.Text(), nullable=False, unique=True),
sa.Column("remote_url", sa.Text()),
sa.Column("branch", sa.String(200), nullable=False, server_default="main"),
sa.Column("status", sa.String(32), nullable=False, server_default="local"),
sa.Column("current_commit", sa.String(64)),
sa.Column("last_pushed_commit", sa.String(64)),
sa.Column("pending_commits", sa.Integer(), nullable=False, server_default="0"),
sa.Column("archived_at", sa.DateTime(timezone=True)),
sa.Column("last_error", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
for column in ("connection_id", "scope", "project_id", "status"):
op.create_index(f"ix_memory_repositories_{column}", "memory_repositories", [column])
op.create_table(
"git_sync_jobs",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("operation_id", sa.String(100), nullable=False, unique=True),
sa.Column(
"repository_id",
sa.String(36),
sa.ForeignKey("memory_repositories.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("resource_id", sa.String(100)),
sa.Column("action", sa.String(40), nullable=False),
sa.Column("summary", sa.String(500), nullable=False),
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
sa.Column("target_commit", sa.String(64)),
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
sa.Column("next_retry_at", sa.DateTime(timezone=True)),
sa.Column("error_message", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
for column in ("operation_id", "repository_id", "resource_id", "status", "next_retry_at"):
op.create_index(f"ix_git_sync_jobs_{column}", "git_sync_jobs", [column])
op.create_table(
"runtime_leases",
sa.Column("name", sa.String(100), primary_key=True),
sa.Column("owner", sa.String(100), nullable=False),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_runtime_leases_owner", "runtime_leases", ["owner"])
op.create_index("ix_runtime_leases_lease_expires_at", "runtime_leases", ["lease_expires_at"])
def downgrade() -> None:
for table in (
"runtime_leases",
"git_sync_jobs",
"memory_repositories",
"git_connections",
"curation_model_calls",
"curation_sources",
"curated_document_revisions",
"curated_documents",
"curation_attempts",
"curation_jobs",
"curation_change_log",
"curation_schedules",
"curation_dependency_state",
"curation_model_configs",
"curation_settings",
):
op.drop_table(table)
with op.batch_alter_table("mcp_tokens") as batch:
batch.drop_constraint("fk_mcp_tokens_usage_profile", type_="foreignkey")
batch.drop_index("ix_mcp_tokens_usage_profile_id")
batch.drop_column("usage_profile_id")
with op.batch_alter_table("memory_references") as batch:
batch.drop_constraint("fk_memory_references_usage_profile", type_="foreignkey")
batch.drop_index("ix_memory_references_usage_profile_id")
batch.drop_column("usage_profile_id")
with op.batch_alter_table("projects") as batch:
batch.drop_index("ix_projects_workspace_type")
batch.drop_column("workspace_type")
op.drop_table("usage_profiles")