[{"data":1,"prerenderedAt":315},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fdata-transfer-workflow":3},{"id":4,"title":5,"body":6,"description":306,"extension":307,"meta":308,"navigation":309,"path":310,"redirect":311,"seo":312,"stem":313,"__hash__":314},"docs\u002Fdocs\u002Fdata-transfer-workflow.md","Data Transfer Workflow",{"type":7,"value":8,"toc":293},"minimark",[9,13,17,22,27,46,50,59,71,75,78,99,103,106,111,116,119,123,126,137,145,149,166,173,177,231,235,242,262,266,289],[10,11,5],"h1",{"id":12},"data-transfer-workflow",[14,15,16],"p",{},"This page describes what happens when a stream runs — from initialization through data transfer to completion.",[18,19,21],"h2",{"id":20},"workflow-phases","Workflow phases",[23,24,26],"h3",{"id":25},"_1-initialization","1. Initialization",[28,29,30,34,37,40,43],"ol",{},[31,32,33],"li",{},"The source reader connects to the source database (or file\u002FS3 storage)",[31,35,36],{},"The target writer connects to the target destination",[31,38,39],{},"Both components connect to NATS JetStream for event delivery",[31,41,42],{},"The reader retrieves table metadata (columns, indexes, keys) from the source",[31,44,45],{},"The writer creates corresponding structures on the target if needed",[23,47,49],{"id":48},"_2-data-reading","2. Data reading",[14,51,52,53,58],{},"The source reader groups rows into bundles (default: 100 rows, configurable in ",[54,55,57],"a",{"href":56},"\u002Fdocs\u002Fstreams\u002Fstream-configuration-guide","stream settings",") and publishes each bundle as an event to NATS.",[14,60,61,62,66,67,70],{},"In ",[63,64,65],"strong",{},"Load mode",", the reader performs direct table reads. In ",[63,68,69],{},"CDC mode",", it captures changes from the database transaction log (PostgreSQL WAL or MySQL binlog).",[23,72,74],{"id":73},"_3-data-transfer","3. Data transfer",[14,76,77],{},"Events flow through NATS JetStream to the target writer:",[79,80,81,87,93],"ul",{},[31,82,83,86],{},[63,84,85],{},"Deduplication"," — each bundle gets a unique key derived from its content; if the same bundle arrives twice (e.g., after a retry), the duplicate is skipped",[31,88,89,92],{},[63,90,91],{},"Ordering"," — ordered events are processed sequentially to maintain consistency; unordered events can be processed in parallel for higher throughput",[31,94,95,98],{},[63,96,97],{},"Persistence"," — NATS persists events so they can be replayed on failure, ensuring no data loss",[23,100,102],{"id":101},"_4-data-writing","4. Data writing",[14,104,105],{},"The target writer processes events using a pool of concurrent workers.",[14,107,61,108,110],{},[63,109,65],{},", the writer performs bulk INSERT (or upsert) operations. For file\u002FS3 targets, data is written as stateful snapshots.",[14,112,61,113,115],{},[63,114,69],{},", the writer also handles UPDATE and DELETE events, matching existing records by primary key.",[14,117,118],{},"Database operations are wrapped in transactions and committed or rolled back as a unit. Failed operations are retried where possible. Streams can be paused and resumed without losing progress.",[23,120,122],{"id":121},"_5-monitoring","5. Monitoring",[14,124,125],{},"Throughout the transfer, the system tracks and reports:",[79,127,128,131,134],{},[31,129,130],{},"Per-table progress: event count, data size, completion percentage",[31,132,133],{},"Overall throughput: total events, processing rate, elapsed time",[31,135,136],{},"Source vs. target event counts to detect processing gaps",[14,138,139,140,144],{},"Progress is streamed to the UI in real time via SSE. See ",[54,141,143],{"href":142},"\u002Fdocs\u002Foperations\u002Fobservability","Observability"," for details on the Monitor tab and logs panel.",[23,146,148],{"id":147},"_6-completion","6. Completion",[28,150,151,154,157,160],{},[31,152,153],{},"The reader signals that all data has been read",[31,155,156],{},"A grace period allows the writer to finish processing remaining events",[31,158,159],{},"Final statistics are reported",[31,161,162,163],{},"The stream status is set to ",[63,164,165],{},"FINISHED",[14,167,168,169,172],{},"If an error occurs, the status is set to ",[63,170,171],{},"FAILED"," with diagnostic details in the logs.",[18,174,176],{"id":175},"data-flow-diagram","Data flow diagram",[178,179,184],"pre",{"className":180,"code":181,"language":182,"meta":183,"style":183},"language-mermaid shiki shiki-themes github-light github-dark","flowchart LR\n    SDB[(Source DB \u002F Files)] --> SR[Source Reader]\n    SR -->|events| NS[NATS JetStream]\n    NS --> W1[Writer 1]\n    NS --> W2[Writer 2]\n    NS --> W3[Writer n]\n    W1 & W2 & W3 --> TDB[(Target DB \u002F Files)]\n","mermaid","",[185,186,187,195,201,207,213,219,225],"code",{"__ignoreMap":183},[188,189,192],"span",{"class":190,"line":191},"line",1,[188,193,194],{},"flowchart LR\n",[188,196,198],{"class":190,"line":197},2,[188,199,200],{},"    SDB[(Source DB \u002F Files)] --> SR[Source Reader]\n",[188,202,204],{"class":190,"line":203},3,[188,205,206],{},"    SR -->|events| NS[NATS JetStream]\n",[188,208,210],{"class":190,"line":209},4,[188,211,212],{},"    NS --> W1[Writer 1]\n",[188,214,216],{"class":190,"line":215},5,[188,217,218],{},"    NS --> W2[Writer 2]\n",[188,220,222],{"class":190,"line":221},6,[188,223,224],{},"    NS --> W3[Writer n]\n",[188,226,228],{"class":190,"line":227},7,[188,229,230],{},"    W1 & W2 & W3 --> TDB[(Target DB \u002F Files)]\n",[18,232,234],{"id":233},"using-data-explorer-in-your-workflow","Using Data Explorer in your workflow",[14,236,237,241],{},[54,238,240],{"href":239},"\u002Fdocs\u002Fdatabase-explorer\u002F","Data Explorer"," can help at each stage:",[79,243,244,250,256],{},[31,245,246,249],{},[63,247,248],{},"Before transfer"," — examine source schema, verify keys and indexes, review sample data",[31,251,252,255],{},[63,253,254],{},"During setup"," — verify target structures were created correctly, compare source and target definitions",[31,257,258,261],{},[63,259,260],{},"After transfer"," — compare data between source and target, check row counts, run validation queries",[18,263,265],{"id":264},"related-docs","Related docs",[79,267,268,274,279,285],{},[31,269,270],{},[54,271,273],{"href":272},"\u002Fdocs\u002Fsystem-architecture","System Architecture",[31,275,276],{},[54,277,278],{"href":56},"Stream Configuration Guide",[31,280,281],{},[54,282,284],{"href":283},"\u002Fdocs\u002Fstreams\u002Fstream-states-lifecycle","Stream States and Lifecycle",[31,286,287],{},[54,288,143],{"href":142},[290,291,292],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":183,"searchDepth":197,"depth":197,"links":294},[295,303,304,305],{"id":20,"depth":197,"text":21,"children":296},[297,298,299,300,301,302],{"id":25,"depth":203,"text":26},{"id":48,"depth":203,"text":49},{"id":73,"depth":203,"text":74},{"id":101,"depth":203,"text":102},{"id":121,"depth":203,"text":122},{"id":147,"depth":203,"text":148},{"id":175,"depth":197,"text":176},{"id":233,"depth":197,"text":234},{"id":264,"depth":197,"text":265},"Learn how data flows from a source to a target destination in DBConvert Streams.","md",{},false,"\u002Fdocs\u002Fdata-transfer-workflow",null,{"title":5,"description":306},"docs\u002Fdata-transfer-workflow","x_-1G-tpvA7L8iTi5cOmBdrOT4nKutNuitBq-Y5O2zs",1783896284664]