[{"data":1,"prerenderedAt":668},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fsystem-architecture":3},{"id":4,"title":5,"body":6,"description":659,"extension":660,"meta":661,"navigation":662,"path":663,"redirect":664,"seo":665,"stem":666,"__hash__":667},"docs\u002Fdocs\u002Fsystem-architecture.md","System Architecture",{"type":7,"value":8,"toc":628},"minimark",[9,13,18,22,26,29,39,69,76,108,115,129,133,136,156,159,180,184,188,191,195,198,256,267,271,275,287,291,302,306,488,492,506,514,518,522,533,537,548,552,556,576,580,603,607,618,624],[10,11,5],"h1",{"id":12},"system-architecture",[14,15,17],"h2",{"id":16},"overview","Overview",[19,20,21],"p",{},"DBConvert Streams is a distributed data processing platform for database migration and real-time replication. The architecture consists of lightweight Go services connected through NATS JetStream, with a Vue.js web UI for management and monitoring.",[14,23,25],{"id":24},"services","Services",[19,27,28],{},"The backend runs as three independent Go binaries:",[30,31,33,34,38],"h3",{"id":32},"api-server-stream-api","API Server (",[35,36,37],"code",{},"stream-api",")",[40,41,42,46,54,57,60,66],"ul",{},[43,44,45],"li",{},"RESTful API for connection, stream, and configuration management",[43,47,48,49],{},"Authentication via ",[50,51,53],"a",{"href":52},"\u002Fdocs\u002Fguide\u002Flicensing","API keys",[43,55,56],{},"Built-in DuckDB engine for file queries and federated queries",[43,58,59],{},"Data Explorer with direct database querying and DuckDB-powered file\u002Fmulti-source SQL",[43,61,62,63,38],{},"Real-time log streaming via SSE (",[35,64,65],{},"\u002Flogs\u002Fstream",[43,67,68],{},"Stream lifecycle orchestration (create, start, stop)",[30,70,72,73,38],{"id":71},"source-reader-stream-reader","Source Reader (",[35,74,75],{},"stream-reader",[40,77,78,81,99,102,105],{},[43,79,80],{},"Reads from source databases and files using specialized adapters",[43,82,83,84],{},"Two reading modes:\n",[40,85,86,93],{},[43,87,88,92],{},[89,90,91],"strong",{},"CDC"," — captures changes from transaction logs (PostgreSQL WAL, MySQL binlog)",[43,94,95,98],{},[89,96,97],{},"Load"," — direct table reads with configurable bundle sizes (default: 100 rows)",[43,100,101],{},"Supports database sources (MySQL 5.7+, PostgreSQL 12+) and file sources (CSV, JSON, Parquet, S3)",[43,103,104],{},"Publishes data events to NATS JetStream",[43,106,107],{},"Reports progress metrics via NATS",[30,109,111,112,38],{"id":110},"target-writer-stream-writer","Target Writer (",[35,113,114],{},"stream-writer",[40,116,117,120,123,126],{},[43,118,119],{},"Consumes data events from NATS JetStream",[43,121,122],{},"Writes to target destinations: MySQL, PostgreSQL, Snowflake, or file formats (CSV, JSONL, Parquet)",[43,124,125],{},"Handles schema creation and cross-database type mapping",[43,127,128],{},"Scales horizontally — multiple writer instances consume in parallel",[14,130,132],{"id":131},"user-interface","User Interface",[19,134,135],{},"The UI is a Vue 3 + TypeScript application available in two forms:",[40,137,138,147],{},[43,139,140,146],{},[89,141,142,143,38],{},"Web UI (",[35,144,145],{},"stream-ui"," — browser-based, served alongside the API or separately",[43,148,149,155],{},[89,150,151,152,38],{},"Desktop app (",[35,153,154],{},"stream-desktop"," — native application built with Wails for Windows, macOS, and Linux; bundles the API server, reader, writer, and NATS into a single process",[19,157,158],{},"Both provide the same feature set:",[40,160,161,164,167,174,177],{},[43,162,163],{},"Connection management and testing",[43,165,166],{},"Stream configuration and monitoring",[43,168,169,173],{},[50,170,172],{"href":171},"\u002Fdocs\u002Fdatabase-explorer\u002F","Data Explorer"," with SQL Console, database diagrams, and federated queries",[43,175,176],{},"Real-time log streaming via SSE",[43,178,179],{},"Run history and usage tracking",[14,181,183],{"id":182},"infrastructure","Infrastructure",[30,185,187],{"id":186},"nats-jetstream","NATS JetStream",[19,189,190],{},"NATS is the backbone for inter-service communication. The reader publishes data events, and writers consume them in parallel. Messages are persisted for fault tolerance and can be replayed on failure.",[30,192,194],{"id":193},"storage-backends","Storage backends",[19,196,197],{},"The active backend depends on the deployment mode:",[199,200,201,217],"table",{},[202,203,204],"thead",{},[205,206,207,211,214],"tr",{},[208,209,210],"th",{},"Function",[208,212,213],{},"Desktop",[208,215,216],{},"Docker (optional)",[218,219,220,235,246],"tbody",{},[205,221,222,226,232],{},[223,224,225],"td",{},"Configuration",[223,227,228,229,38],{},"SQLite (",[35,230,231],{},"\u003CappRoot>\u002Fconfig\u002Fconfig.db",[223,233,234],{},"Consul KV",[205,236,237,240,243],{},[223,238,239],{},"Secrets",[223,241,242],{},"Encrypted file (XChaCha20-Poly1305)",[223,244,245],{},"HashiCorp Vault",[205,247,248,251,254],{},[223,249,250],{},"Service coordination",[223,252,253],{},"NATS",[223,255,253],{},[19,257,258,259,262,263,266],{},"Backends are selected via environment variables: ",[35,260,261],{},"CONFIG_BACKEND",", ",[35,264,265],{},"SECRETS_BACKEND",".",[14,268,270],{"id":269},"data-flow","Data flow",[30,272,274],{"id":273},"_1-initialization","1. Initialization",[276,277,278,281,284],"ol",{},[43,279,280],{},"Reader retrieves table metadata (columns, indexes, keys) from the source",[43,282,283],{},"Metadata is published to NATS for the target to create corresponding structures",[43,285,286],{},"Writer creates tables\u002Fschemas on the target where applicable",[30,288,290],{"id":289},"_2-transfer","2. Transfer",[276,292,293,296,299],{},[43,294,295],{},"Reader reads data in configurable bundles and publishes to NATS JetStream",[43,297,298],{},"Writers consume events in parallel and write to the target",[43,300,301],{},"Progress metrics are reported via NATS and streamed to the UI via SSE",[30,303,305],{"id":304},"data-flow-diagram","Data flow diagram",[307,308,313],"pre",{"className":309,"code":310,"language":311,"meta":312,"style":312},"language-mermaid shiki shiki-themes github-light github-dark","flowchart TB\n    subgraph Control[Control Plane]\n        UI[Web UI]\n        API[API Server]\n        UI --> API\n    end\n\n    subgraph DataFlow[Data Flow]\n        SDB[(Source DB \u002F Files)]\n        SR[Source Reader]\n        NS[NATS JetStream]\n        W1[Writer 1]\n        W2[Writer 2]\n        W3[Writer n]\n        TDB[(Target DB \u002F Files)]\n\n        SDB --> SR\n        SR --> NS\n        NS --> W1 & W2 & W3\n        W1 & W2 & W3 --> TDB\n    end\n\n    %% Control paths\n    API -->|Start \u002F Stop| SR\n\n    %% Monitoring paths\n    W1 & W2 & W3 -->|metrics| NS\n    SR -->|metrics| NS\n    NS -->|SSE| API\n","mermaid","",[35,314,315,323,329,335,341,347,353,360,366,372,378,384,390,396,402,408,413,419,425,431,437,442,447,453,459,464,470,476,482],{"__ignoreMap":312},[316,317,320],"span",{"class":318,"line":319},"line",1,[316,321,322],{},"flowchart TB\n",[316,324,326],{"class":318,"line":325},2,[316,327,328],{},"    subgraph Control[Control Plane]\n",[316,330,332],{"class":318,"line":331},3,[316,333,334],{},"        UI[Web UI]\n",[316,336,338],{"class":318,"line":337},4,[316,339,340],{},"        API[API Server]\n",[316,342,344],{"class":318,"line":343},5,[316,345,346],{},"        UI --> API\n",[316,348,350],{"class":318,"line":349},6,[316,351,352],{},"    end\n",[316,354,356],{"class":318,"line":355},7,[316,357,359],{"emptyLinePlaceholder":358},true,"\n",[316,361,363],{"class":318,"line":362},8,[316,364,365],{},"    subgraph DataFlow[Data Flow]\n",[316,367,369],{"class":318,"line":368},9,[316,370,371],{},"        SDB[(Source DB \u002F Files)]\n",[316,373,375],{"class":318,"line":374},10,[316,376,377],{},"        SR[Source Reader]\n",[316,379,381],{"class":318,"line":380},11,[316,382,383],{},"        NS[NATS JetStream]\n",[316,385,387],{"class":318,"line":386},12,[316,388,389],{},"        W1[Writer 1]\n",[316,391,393],{"class":318,"line":392},13,[316,394,395],{},"        W2[Writer 2]\n",[316,397,399],{"class":318,"line":398},14,[316,400,401],{},"        W3[Writer n]\n",[316,403,405],{"class":318,"line":404},15,[316,406,407],{},"        TDB[(Target DB \u002F Files)]\n",[316,409,411],{"class":318,"line":410},16,[316,412,359],{"emptyLinePlaceholder":358},[316,414,416],{"class":318,"line":415},17,[316,417,418],{},"        SDB --> SR\n",[316,420,422],{"class":318,"line":421},18,[316,423,424],{},"        SR --> NS\n",[316,426,428],{"class":318,"line":427},19,[316,429,430],{},"        NS --> W1 & W2 & W3\n",[316,432,434],{"class":318,"line":433},20,[316,435,436],{},"        W1 & W2 & W3 --> TDB\n",[316,438,440],{"class":318,"line":439},21,[316,441,352],{},[316,443,445],{"class":318,"line":444},22,[316,446,359],{"emptyLinePlaceholder":358},[316,448,450],{"class":318,"line":449},23,[316,451,452],{},"    %% Control paths\n",[316,454,456],{"class":318,"line":455},24,[316,457,458],{},"    API -->|Start \u002F Stop| SR\n",[316,460,462],{"class":318,"line":461},25,[316,463,359],{"emptyLinePlaceholder":358},[316,465,467],{"class":318,"line":466},26,[316,468,469],{},"    %% Monitoring paths\n",[316,471,473],{"class":318,"line":472},27,[316,474,475],{},"    W1 & W2 & W3 -->|metrics| NS\n",[316,477,479],{"class":318,"line":478},28,[316,480,481],{},"    SR -->|metrics| NS\n",[316,483,485],{"class":318,"line":484},29,[316,486,487],{},"    NS -->|SSE| API\n",[14,489,491],{"id":490},"security","Security",[40,493,494,497,500,503],{},[43,495,496],{},"API key validation for all service access",[43,498,499],{},"SSL\u002FTLS encryption for database connections and data in transit",[43,501,502],{},"Encrypted credential storage (local encrypted file or Vault, depending on deployment mode)",[43,504,505],{},"Client certificate support for database connections",[19,507,508,509,513],{},"See ",[50,510,512],{"href":511},"\u002Fdocs\u002Fnetwork-security\u002Fssl-configuration","SSL Configuration"," for details.",[14,515,517],{"id":516},"deployment-options","Deployment options",[30,519,521],{"id":520},"desktop-application","Desktop application",[40,523,524,527,530],{},[43,525,526],{},"Lightweight stack: SQLite + encrypted secrets file + embedded NATS",[43,528,529],{},"No external services required",[43,531,532],{},"Available for Windows, macOS, and Linux",[30,534,536],{"id":535},"docker","Docker",[40,538,539,542,545],{},[43,540,541],{},"Complete docker-compose configurations with NATS included",[43,543,544],{},"Optional Consul and Vault for distributed deployments",[43,546,547],{},"Runs on any platform that supports Docker: AWS, GCP, Azure, DigitalOcean, on-premises",[14,549,551],{"id":550},"database-support","Database support",[30,553,555],{"id":554},"sources","Sources",[40,557,558,564,570],{},[43,559,560,563],{},[89,561,562],{},"MySQL"," 5.7+ — full table snapshots, CDC via binlog",[43,565,566,569],{},[89,567,568],{},"PostgreSQL"," 12+ — full snapshots, CDC via logical replication",[43,571,572,575],{},[89,573,574],{},"Files"," — CSV, JSON, Parquet (local or S3-compatible storage)",[30,577,579],{"id":578},"targets","Targets",[40,581,582,587,592,598],{},[43,583,584,586],{},[89,585,562],{}," 5.7+ — direct writes with transaction support",[43,588,589,591],{},[89,590,568],{}," 12+ — direct writes with conflict resolution",[43,593,594,597],{},[89,595,596],{},"Snowflake"," — batch loading with staging tables",[43,599,600,602],{},[89,601,574],{}," — CSV, JSONL, Parquet output formats",[30,604,606],{"id":605},"requirements","Requirements",[40,608,609,612,615],{},[43,610,611],{},"MySQL CDC requires binary logging enabled",[43,613,614],{},"PostgreSQL CDC requires logical replication configuration",[43,616,617],{},"Appropriate database permissions needed for CDC\u002Freplication",[19,619,508,620,623],{},[50,621,622],{"href":52},"Licensing, Seats, and Telemetry"," for API key and access details.",[625,626,627],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":312,"searchDepth":325,"depth":325,"links":629},[630,631,639,640,644,649,650,654],{"id":16,"depth":325,"text":17},{"id":24,"depth":325,"text":25,"children":632},[633,635,637],{"id":32,"depth":331,"text":634},"API Server (stream-api)",{"id":71,"depth":331,"text":636},"Source Reader (stream-reader)",{"id":110,"depth":331,"text":638},"Target Writer (stream-writer)",{"id":131,"depth":325,"text":132},{"id":182,"depth":325,"text":183,"children":641},[642,643],{"id":186,"depth":331,"text":187},{"id":193,"depth":331,"text":194},{"id":269,"depth":325,"text":270,"children":645},[646,647,648],{"id":273,"depth":331,"text":274},{"id":289,"depth":331,"text":290},{"id":304,"depth":331,"text":305},{"id":490,"depth":325,"text":491},{"id":516,"depth":325,"text":517,"children":651},[652,653],{"id":520,"depth":331,"text":521},{"id":535,"depth":331,"text":536},{"id":550,"depth":325,"text":551,"children":655},[656,657,658],{"id":554,"depth":331,"text":555},{"id":578,"depth":331,"text":579},{"id":605,"depth":331,"text":606},"System Architecture of DBConvert Streams","md",{},false,"\u002Fdocs\u002Fsystem-architecture",null,{"title":5,"description":659},"docs\u002Fsystem-architecture","t4nZiiwG1L2-tzL8xBmmWuRTpyeE4ZfSHGAO0AiQtRI",1783896283760]