[{"data":1,"prerenderedAt":688},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fstreams\u002Fstream-configuration-overview":3},{"id":4,"title":5,"body":6,"description":679,"extension":680,"meta":681,"navigation":682,"path":683,"redirect":684,"seo":685,"stem":686,"__hash__":687},"docs\u002Fdocs\u002Fstreams\u002Fstream-configuration-overview.md","Stream Configuration Overview",{"type":7,"value":8,"toc":671},"minimark",[9,13,17,20,51,62,67,70,212,215,251,255,496,499,536,540,547,567,571,603,607,610,639,643,667],[10,11,5],"h1",{"id":12},"stream-configuration-overview",[14,15,16],"p",{},"Use this page when you need to understand how a stream configuration is organized before looking up individual field paths.",[14,18,19],{},"A stream configuration defines:",[21,22,23,31,37],"ul",{},[24,25,26,27],"li",{},"what to read in ",[28,29,30],"code",{},"source",[24,32,33,34],{},"where to write in ",[28,35,36],{},"target",[24,38,39,40,43,44,47,48],{},"how to control execution with ",[28,41,42],{},"mode",", ",[28,45,46],{},"reportingInterval",", and optional ",[28,49,50],{},"limits",[14,52,53,54,57,58,61],{},"Saved configurations add metadata such as ",[28,55,56],{},"id"," and ",[28,59,60],{},"created",", but the working shape stays the same.",[63,64,66],"h2",{"id":65},"core-structure","Core structure",[14,68,69],{},"Every stream configuration follows the same top-level pattern:",[71,72,77],"pre",{"className":73,"code":74,"language":75,"meta":76,"style":76},"language-json shiki shiki-themes github-light github-dark","{\n  \"name\": \"My stream\",\n  \"mode\": \"load\",\n  \"reportingInterval\": 5,\n  \"source\": {\n    \"connections\": [],\n    \"options\": {}\n  },\n  \"target\": {\n    \"id\": \"conn_target\",\n    \"spec\": {}\n  },\n  \"limits\": {}\n}\n","json","",[28,78,79,88,105,118,131,140,149,158,164,172,185,193,198,206],{"__ignoreMap":76},[80,81,84],"span",{"class":82,"line":83},"line",1,[80,85,87],{"class":86},"sVt8B","{\n",[80,89,91,95,98,102],{"class":82,"line":90},2,[80,92,94],{"class":93},"sj4cs","  \"name\"",[80,96,97],{"class":86},": ",[80,99,101],{"class":100},"sZZnC","\"My stream\"",[80,103,104],{"class":86},",\n",[80,106,108,111,113,116],{"class":82,"line":107},3,[80,109,110],{"class":93},"  \"mode\"",[80,112,97],{"class":86},[80,114,115],{"class":100},"\"load\"",[80,117,104],{"class":86},[80,119,121,124,126,129],{"class":82,"line":120},4,[80,122,123],{"class":93},"  \"reportingInterval\"",[80,125,97],{"class":86},[80,127,128],{"class":93},"5",[80,130,104],{"class":86},[80,132,134,137],{"class":82,"line":133},5,[80,135,136],{"class":93},"  \"source\"",[80,138,139],{"class":86},": {\n",[80,141,143,146],{"class":82,"line":142},6,[80,144,145],{"class":93},"    \"connections\"",[80,147,148],{"class":86},": [],\n",[80,150,152,155],{"class":82,"line":151},7,[80,153,154],{"class":93},"    \"options\"",[80,156,157],{"class":86},": {}\n",[80,159,161],{"class":82,"line":160},8,[80,162,163],{"class":86},"  },\n",[80,165,167,170],{"class":82,"line":166},9,[80,168,169],{"class":93},"  \"target\"",[80,171,139],{"class":86},[80,173,175,178,180,183],{"class":82,"line":174},10,[80,176,177],{"class":93},"    \"id\"",[80,179,97],{"class":86},[80,181,182],{"class":100},"\"conn_target\"",[80,184,104],{"class":86},[80,186,188,191],{"class":82,"line":187},11,[80,189,190],{"class":93},"    \"spec\"",[80,192,157],{"class":86},[80,194,196],{"class":82,"line":195},12,[80,197,163],{"class":86},[80,199,201,204],{"class":82,"line":200},13,[80,202,203],{"class":93},"  \"limits\"",[80,205,157],{"class":86},[80,207,209],{"class":82,"line":208},14,[80,210,211],{"class":86},"}\n",[14,213,214],{},"Mental model:",[21,216,217,223,233,246],{},[24,218,219,222],{},[28,220,221],{},"source.connections[]"," is always an array, even for a single source",[24,224,225,226,229,230],{},"each source connection chooses one source family: database fields, ",[28,227,228],{},"s3",", or ",[28,231,232],{},"files",[24,234,235,238,239,43,242,229,244],{},[28,236,237],{},"target.spec"," always contains exactly one target family: ",[28,240,241],{},"db",[28,243,232],{},[28,245,228],{},[24,247,248,250],{},[28,249,50],{}," is optional and is mainly used for bounded runs and validation",[63,252,254],{"id":253},"annotated-example","Annotated example",[71,256,258],{"className":73,"code":257,"language":75,"meta":76,"style":76},"{\n  \"name\": \"PostgreSQL to MySQL\",\n  \"mode\": \"load\",\n  \"reportingInterval\": 3,\n  \"source\": {\n    \"connections\": [\n      {\n        \"connectionId\": \"conn_postgres_prod\",\n        \"database\": \"postgres\",\n        \"tables\": [\n          { \"name\": \"public.actor\" }\n        ]\n      }\n    ],\n    \"options\": {\n      \"dataBundleSize\": 1000\n    }\n  },\n  \"target\": {\n    \"id\": \"conn_mysql_target\",\n    \"spec\": {\n      \"db\": {\n        \"database\": \"testdb\",\n        \"schemaPolicy\": \"create_missing_only\",\n        \"writeMode\": \"upsert\"\n      }\n    }\n  }\n}\n",[28,259,260,264,275,285,296,302,309,314,326,338,345,361,366,371,376,383,394,400,405,412,424,431,439,451,464,475,480,485,491],{"__ignoreMap":76},[80,261,262],{"class":82,"line":83},[80,263,87],{"class":86},[80,265,266,268,270,273],{"class":82,"line":90},[80,267,94],{"class":93},[80,269,97],{"class":86},[80,271,272],{"class":100},"\"PostgreSQL to MySQL\"",[80,274,104],{"class":86},[80,276,277,279,281,283],{"class":82,"line":107},[80,278,110],{"class":93},[80,280,97],{"class":86},[80,282,115],{"class":100},[80,284,104],{"class":86},[80,286,287,289,291,294],{"class":82,"line":120},[80,288,123],{"class":93},[80,290,97],{"class":86},[80,292,293],{"class":93},"3",[80,295,104],{"class":86},[80,297,298,300],{"class":82,"line":133},[80,299,136],{"class":93},[80,301,139],{"class":86},[80,303,304,306],{"class":82,"line":142},[80,305,145],{"class":93},[80,307,308],{"class":86},": [\n",[80,310,311],{"class":82,"line":151},[80,312,313],{"class":86},"      {\n",[80,315,316,319,321,324],{"class":82,"line":160},[80,317,318],{"class":93},"        \"connectionId\"",[80,320,97],{"class":86},[80,322,323],{"class":100},"\"conn_postgres_prod\"",[80,325,104],{"class":86},[80,327,328,331,333,336],{"class":82,"line":166},[80,329,330],{"class":93},"        \"database\"",[80,332,97],{"class":86},[80,334,335],{"class":100},"\"postgres\"",[80,337,104],{"class":86},[80,339,340,343],{"class":82,"line":174},[80,341,342],{"class":93},"        \"tables\"",[80,344,308],{"class":86},[80,346,347,350,353,355,358],{"class":82,"line":187},[80,348,349],{"class":86},"          { ",[80,351,352],{"class":93},"\"name\"",[80,354,97],{"class":86},[80,356,357],{"class":100},"\"public.actor\"",[80,359,360],{"class":86}," }\n",[80,362,363],{"class":82,"line":195},[80,364,365],{"class":86},"        ]\n",[80,367,368],{"class":82,"line":200},[80,369,370],{"class":86},"      }\n",[80,372,373],{"class":82,"line":208},[80,374,375],{"class":86},"    ],\n",[80,377,379,381],{"class":82,"line":378},15,[80,380,154],{"class":93},[80,382,139],{"class":86},[80,384,386,389,391],{"class":82,"line":385},16,[80,387,388],{"class":93},"      \"dataBundleSize\"",[80,390,97],{"class":86},[80,392,393],{"class":93},"1000\n",[80,395,397],{"class":82,"line":396},17,[80,398,399],{"class":86},"    }\n",[80,401,403],{"class":82,"line":402},18,[80,404,163],{"class":86},[80,406,408,410],{"class":82,"line":407},19,[80,409,169],{"class":93},[80,411,139],{"class":86},[80,413,415,417,419,422],{"class":82,"line":414},20,[80,416,177],{"class":93},[80,418,97],{"class":86},[80,420,421],{"class":100},"\"conn_mysql_target\"",[80,423,104],{"class":86},[80,425,427,429],{"class":82,"line":426},21,[80,428,190],{"class":93},[80,430,139],{"class":86},[80,432,434,437],{"class":82,"line":433},22,[80,435,436],{"class":93},"      \"db\"",[80,438,139],{"class":86},[80,440,442,444,446,449],{"class":82,"line":441},23,[80,443,330],{"class":93},[80,445,97],{"class":86},[80,447,448],{"class":100},"\"testdb\"",[80,450,104],{"class":86},[80,452,454,457,459,462],{"class":82,"line":453},24,[80,455,456],{"class":93},"        \"schemaPolicy\"",[80,458,97],{"class":86},[80,460,461],{"class":100},"\"create_missing_only\"",[80,463,104],{"class":86},[80,465,467,470,472],{"class":82,"line":466},25,[80,468,469],{"class":93},"        \"writeMode\"",[80,471,97],{"class":86},[80,473,474],{"class":100},"\"upsert\"\n",[80,476,478],{"class":82,"line":477},26,[80,479,370],{"class":86},[80,481,483],{"class":82,"line":482},27,[80,484,399],{"class":86},[80,486,488],{"class":82,"line":487},28,[80,489,490],{"class":86},"  }\n",[80,492,494],{"class":82,"line":493},29,[80,495,211],{"class":86},[14,497,498],{},"Read it as:",[21,500,501,512,518,524,530],{},[24,502,503,43,506,508,509,511],{},[28,504,505],{},"name",[28,507,42],{},", and ",[28,510,46],{}," define the run at the top level",[24,513,514,517],{},[28,515,516],{},"source.connections[0]"," selects one stored source connection and scopes what to read from it",[24,519,520,523],{},[28,521,522],{},"source.options"," adds execution settings that belong to the read side",[24,525,526,529],{},[28,527,528],{},"target.id"," selects the destination connection",[24,531,532,535],{},[28,533,534],{},"target.spec.db"," tells DBConvert Streams to write into a database target",[63,537,539],{"id":538},"source-vs-target","Source vs target",[14,541,542,57,544,546],{},[28,543,30],{},[28,545,36],{}," are intentionally asymmetric:",[21,548,549,554,557,562],{},[24,550,551,553],{},[28,552,221],{}," can contain one or more connections",[24,555,556],{},"each source connection carries its own scope, such as selected tables, a manifest path, or local file paths",[24,558,559,561],{},[28,560,36],{}," always points to one destination connection at a time",[24,563,564,566],{},[28,565,237],{}," decides how that destination should be written to",[63,568,570],{"id":569},"which-page-to-open-next","Which page to open next",[21,572,573,582,589,596],{},[24,574,575,576,581],{},"Use ",[577,578,580],"a",{"href":579},"\u002Fdocs\u002Fstreams\u002Fsource-configuration","Source Configuration"," when you need to define tables, files, queries, or S3 scope.",[24,583,575,584,588],{},[577,585,587],{"href":586},"\u002Fdocs\u002Fstreams\u002Ftarget-configuration","Target Configuration"," when you need to define database writes, file formats, staging, or upload behavior.",[24,590,575,591,595],{},[577,592,594],{"href":593},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Stream Configuration Reference"," when you need exact field paths.",[24,597,575,598,602],{},[577,599,601],{"href":600},"\u002Fdocs\u002Fstreams\u002Fstream-examples","Stream Configuration Examples"," when you want full copyable payloads.",[63,604,606],{"id":605},"verification","Verification",[14,608,609],{},"Before saving or using a payload:",[21,611,612,622,625,631],{},[24,613,614,615,617,618,621],{},"confirm it uses ",[28,616,221],{}," rather than an old flat ",[28,619,620],{},"source.id"," shape",[24,623,624],{},"confirm each source connection uses only one source family",[24,626,627,628,630],{},"confirm ",[28,629,237],{}," contains exactly one family object",[24,632,633,634,57,636,638],{},"confirm saved configs add metadata such as ",[28,635,56],{},[28,637,60],{},", but your working payload does not depend on those fields",[63,640,642],{"id":641},"related-docs","Related docs",[21,644,645,651,655,659,663],{},[24,646,647],{},[577,648,650],{"href":649},"\u002Fdocs\u002Fstreams\u002Fstream-configuration-guide","Stream Configuration Guide",[24,652,653],{},[577,654,580],{"href":579},[24,656,657],{},[577,658,587],{"href":586},[24,660,661],{},[577,662,594],{"href":593},[24,664,665],{},[577,666,601],{"href":600},[668,669,670],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":76,"searchDepth":90,"depth":90,"links":672},[673,674,675,676,677,678],{"id":65,"depth":90,"text":66},{"id":253,"depth":90,"text":254},{"id":538,"depth":90,"text":539},{"id":569,"depth":90,"text":570},{"id":605,"depth":90,"text":606},{"id":641,"depth":90,"text":642},"Understand how a DBConvert Streams configuration is organized before looking up exact field paths.","md",{},false,"\u002Fdocs\u002Fstreams\u002Fstream-configuration-overview",null,{"title":5,"description":679},"docs\u002Fstreams\u002Fstream-configuration-overview","rCsn-gPaD03Yw4sy3MmeBximEC2T6JcgqUDyVL9sunA",1783896284825]