[{"data":1,"prerenderedAt":759},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fstreams\u002Fsource-configuration":3},{"id":4,"title":5,"body":6,"description":750,"extension":751,"meta":752,"navigation":753,"path":754,"redirect":755,"seo":756,"stem":757,"__hash__":758},"docs\u002Fdocs\u002Fstreams\u002Fsource-configuration.md","Source Configuration",{"type":7,"value":8,"toc":739},"minimark",[9,13,17,25,30,33,66,75,79,82,85,184,187,221,225,236,239,265,273,277,283,285,365,367,384,388,393,395,454,457,483,495,499,504,506,577,579,598,602,608,611,642,653,660,664,667,699,703,735],[10,11,5],"h1",{"id":12},"source-configuration",[14,15,16],"p",{},"Use this page when you need to define what a stream reads from and how that source is scoped.",[14,18,19,20,24],{},"Every source configuration starts in ",[21,22,23],"code",{},"source.connections[]",". Each entry combines a stored connection ID with stream-specific scope such as selected tables, custom queries, S3 paths, or local file paths.",[26,27,29],"h2",{"id":28},"source-families","Source families",[14,31,32],{},"Use one source family per connection entry:",[34,35,36,54,60],"ul",{},[37,38,39,40,43,44,43,47,50,51],"li",{},"database fields such as ",[21,41,42],{},"database",", ",[21,45,46],{},"schema",[21,48,49],{},"tables",", and ",[21,52,53],{},"queries",[37,55,56,59],{},[21,57,58],{},"s3"," for S3-compatible object storage",[37,61,62,65],{},[21,63,64],{},"files"," for local file sources",[14,67,68,69,71,72,74],{},"Do not mix database fields with ",[21,70,58],{}," or ",[21,73,64],{}," inside the same connection entry.",[26,76,78],{"id":77},"database-sources","Database sources",[14,80,81],{},"Use database fields when the source is MySQL or PostgreSQL and the stream should read tables or custom queries from that connection.",[14,83,84],{},"Typical shape:",[86,87,92],"pre",{"className":88,"code":89,"language":90,"meta":91,"style":91},"language-json shiki shiki-themes github-light github-dark","{\n  \"connectionId\": \"conn_postgres_prod\",\n  \"database\": \"sakila\",\n  \"schema\": \"public\",\n  \"tables\": [\n    { \"name\": \"actor\" }\n  ]\n}\n","json","",[21,93,94,103,120,133,146,155,172,178],{"__ignoreMap":91},[95,96,99],"span",{"class":97,"line":98},"line",1,[95,100,102],{"class":101},"sVt8B","{\n",[95,104,106,110,113,117],{"class":97,"line":105},2,[95,107,109],{"class":108},"sj4cs","  \"connectionId\"",[95,111,112],{"class":101},": ",[95,114,116],{"class":115},"sZZnC","\"conn_postgres_prod\"",[95,118,119],{"class":101},",\n",[95,121,123,126,128,131],{"class":97,"line":122},3,[95,124,125],{"class":108},"  \"database\"",[95,127,112],{"class":101},[95,129,130],{"class":115},"\"sakila\"",[95,132,119],{"class":101},[95,134,136,139,141,144],{"class":97,"line":135},4,[95,137,138],{"class":108},"  \"schema\"",[95,140,112],{"class":101},[95,142,143],{"class":115},"\"public\"",[95,145,119],{"class":101},[95,147,149,152],{"class":97,"line":148},5,[95,150,151],{"class":108},"  \"tables\"",[95,153,154],{"class":101},": [\n",[95,156,158,161,164,166,169],{"class":97,"line":157},6,[95,159,160],{"class":101},"    { ",[95,162,163],{"class":108},"\"name\"",[95,165,112],{"class":101},[95,167,168],{"class":115},"\"actor\"",[95,170,171],{"class":101}," }\n",[95,173,175],{"class":97,"line":174},7,[95,176,177],{"class":101},"  ]\n",[95,179,181],{"class":97,"line":180},8,[95,182,183],{"class":101},"}\n",[14,185,186],{},"Key points:",[34,188,189,195,200,205,210,215],{},[37,190,191,194],{},[21,192,193],{},"connectionId"," selects the stored source connection",[37,196,197,199],{},[21,198,42],{}," chooses the database on that connection",[37,201,202,204],{},[21,203,46],{}," is optional and mainly relevant for database sources",[37,206,207,209],{},[21,208,49],{}," selects physical tables",[37,211,212,214],{},[21,213,53],{}," defines virtual query sources and is supported in Load mode only",[37,216,217,220],{},[21,218,219],{},"alias"," is only needed when the stream reads from multiple source connections",[26,222,224],{"id":223},"table-selection","Table selection",[14,226,227,228,231,232,235],{},"Use ",[21,229,230],{},"tables[].selection"," when a table source needs a narrower scope than ",[21,233,234],{},"SELECT *",".",[14,237,238],{},"The selection object controls:",[34,240,241,247,253,259],{},[37,242,243,246],{},[21,244,245],{},"selectedColumns"," for column projection",[37,248,249,252],{},[21,250,251],{},"filters"," for row predicates",[37,254,255,258],{},[21,256,257],{},"sorts"," for ordering",[37,260,261,264],{},[21,262,263],{},"limit"," for table-level row caps",[14,266,267,268,235],{},"For operator behavior and UI mapping, use ",[269,270,272],"a",{"href":271},"\u002Fdocs\u002Fstreams\u002Fdata-filter-guide","Data Filter Guide",[26,274,276],{"id":275},"query-sources","Query sources",[14,278,227,279,282],{},[21,280,281],{},"queries[]"," when the source should be defined by SQL instead of by selected physical tables.",[14,284,84],{},[86,286,288],{"className":88,"code":287,"language":90,"meta":91,"style":91},"{\n  \"connectionId\": \"conn_mysql_prod\",\n  \"database\": \"commerce\",\n  \"queries\": [\n    {\n      \"name\": \"recent_orders\",\n      \"query\": \"SELECT * FROM orders WHERE created_at >= '2024-01-01'\"\n    }\n  ]\n}\n",[21,289,290,294,305,316,323,328,340,350,355,360],{"__ignoreMap":91},[95,291,292],{"class":97,"line":98},[95,293,102],{"class":101},[95,295,296,298,300,303],{"class":97,"line":105},[95,297,109],{"class":108},[95,299,112],{"class":101},[95,301,302],{"class":115},"\"conn_mysql_prod\"",[95,304,119],{"class":101},[95,306,307,309,311,314],{"class":97,"line":122},[95,308,125],{"class":108},[95,310,112],{"class":101},[95,312,313],{"class":115},"\"commerce\"",[95,315,119],{"class":101},[95,317,318,321],{"class":97,"line":135},[95,319,320],{"class":108},"  \"queries\"",[95,322,154],{"class":101},[95,324,325],{"class":97,"line":148},[95,326,327],{"class":101},"    {\n",[95,329,330,333,335,338],{"class":97,"line":157},[95,331,332],{"class":108},"      \"name\"",[95,334,112],{"class":101},[95,336,337],{"class":115},"\"recent_orders\"",[95,339,119],{"class":101},[95,341,342,345,347],{"class":97,"line":174},[95,343,344],{"class":108},"      \"query\"",[95,346,112],{"class":101},[95,348,349],{"class":115},"\"SELECT * FROM orders WHERE created_at >= '2024-01-01'\"\n",[95,351,352],{"class":97,"line":180},[95,353,354],{"class":101},"    }\n",[95,356,358],{"class":97,"line":357},9,[95,359,177],{"class":101},[95,361,363],{"class":97,"line":362},10,[95,364,183],{"class":101},[14,366,186],{},[34,368,369,372,378],{},[37,370,371],{},"query sources are Load mode only",[37,373,374,377],{},[21,375,376],{},"name"," becomes the virtual source name used by the target side",[37,379,380,383],{},[21,381,382],{},"columns[]"," is optional and is mainly used as a schema hint",[26,385,387],{"id":386},"s3-sources","S3 sources",[14,389,227,390,392],{},[21,391,58],{}," when the source connection points to AWS S3, DigitalOcean Spaces, MinIO, or another S3-compatible provider.",[14,394,84],{},[86,396,398],{"className":88,"code":397,"language":90,"meta":91,"style":91},"{\n  \"connectionId\": \"conn_s3_exports\",\n  \"s3\": {\n    \"bucket\": \"company-data\",\n    \"manifestPath\": \"s3:\u002F\u002Fcompany-data\u002Fexports\u002Forders\u002Fmanifest.json\"\n  }\n}\n",[21,399,400,404,415,423,435,445,450],{"__ignoreMap":91},[95,401,402],{"class":97,"line":98},[95,403,102],{"class":101},[95,405,406,408,410,413],{"class":97,"line":105},[95,407,109],{"class":108},[95,409,112],{"class":101},[95,411,412],{"class":115},"\"conn_s3_exports\"",[95,414,119],{"class":101},[95,416,417,420],{"class":97,"line":122},[95,418,419],{"class":108},"  \"s3\"",[95,421,422],{"class":101},": {\n",[95,424,425,428,430,433],{"class":97,"line":135},[95,426,427],{"class":108},"    \"bucket\"",[95,429,112],{"class":101},[95,431,432],{"class":115},"\"company-data\"",[95,434,119],{"class":101},[95,436,437,440,442],{"class":97,"line":148},[95,438,439],{"class":108},"    \"manifestPath\"",[95,441,112],{"class":101},[95,443,444],{"class":115},"\"s3:\u002F\u002Fcompany-data\u002Fexports\u002Forders\u002Fmanifest.json\"\n",[95,446,447],{"class":97,"line":157},[95,448,449],{"class":101},"  }\n",[95,451,452],{"class":97,"line":174},[95,453,183],{"class":101},[14,455,456],{},"You can scope an S3 source with:",[34,458,459,469,477],{},[37,460,461,464,465,468],{},[21,462,463],{},"bucket"," plus ",[21,466,467],{},"prefixes[]"," for recursive folder-style selection",[37,470,471,464,473,476],{},[21,472,463],{},[21,474,475],{},"objects[]"," for exact keys",[37,478,479,482],{},[21,480,481],{},"manifestPath"," for manifest-driven selection",[14,484,485,486,488,489,71,492,235],{},"Do not combine ",[21,487,481],{}," with ",[21,490,491],{},"prefixes",[21,493,494],{},"objects",[26,496,498],{"id":497},"local-file-sources","Local file sources",[14,500,227,501,503],{},[21,502,64],{}," when the source is a local filesystem connection.",[14,505,84],{},[86,507,509],{"className":88,"code":508,"language":90,"meta":91,"style":91},"{\n  \"connectionId\": \"conn_local_files\",\n  \"files\": {\n    \"basePath\": \"\u002Fdata\u002Fimports\",\n    \"paths\": [\n      \"orders\u002F2026-03\u002Forders.csv\",\n      \"customers\u002Fcustomers.parquet\"\n    ]\n  }\n}\n",[21,510,511,515,526,533,545,552,559,564,569,573],{"__ignoreMap":91},[95,512,513],{"class":97,"line":98},[95,514,102],{"class":101},[95,516,517,519,521,524],{"class":97,"line":105},[95,518,109],{"class":108},[95,520,112],{"class":101},[95,522,523],{"class":115},"\"conn_local_files\"",[95,525,119],{"class":101},[95,527,528,531],{"class":97,"line":122},[95,529,530],{"class":108},"  \"files\"",[95,532,422],{"class":101},[95,534,535,538,540,543],{"class":97,"line":135},[95,536,537],{"class":108},"    \"basePath\"",[95,539,112],{"class":101},[95,541,542],{"class":115},"\"\u002Fdata\u002Fimports\"",[95,544,119],{"class":101},[95,546,547,550],{"class":97,"line":148},[95,548,549],{"class":108},"    \"paths\"",[95,551,154],{"class":101},[95,553,554,557],{"class":97,"line":157},[95,555,556],{"class":115},"      \"orders\u002F2026-03\u002Forders.csv\"",[95,558,119],{"class":101},[95,560,561],{"class":97,"line":174},[95,562,563],{"class":115},"      \"customers\u002Fcustomers.parquet\"\n",[95,565,566],{"class":97,"line":180},[95,567,568],{"class":101},"    ]\n",[95,570,571],{"class":97,"line":357},[95,572,449],{"class":101},[95,574,575],{"class":97,"line":362},[95,576,183],{"class":101},[14,578,186],{},[34,580,581,587,595],{},[37,582,583,586],{},[21,584,585],{},"basePath"," is the root directory for that file connection",[37,588,589,592,593],{},[21,590,591],{},"paths[]"," are relative to ",[21,594,585],{},[37,596,597],{},"file-based streams use the connection's filesystem context, not the machine that saved the config",[26,599,601],{"id":600},"source-options","Source options",[14,603,604,607],{},[21,605,606],{},"source.options"," holds execution settings that belong to the read side.",[14,609,610],{},"Important fields:",[34,612,613,619,625,631,636],{},[37,614,615,618],{},[21,616,617],{},"dataBundleSize"," for batch sizing in both Load and CDC workflows",[37,620,621,624],{},[21,622,623],{},"operations"," for CDC event types to capture",[37,626,627,630],{},[21,628,629],{},"replicationSlot"," for PostgreSQL CDC",[37,632,633,630],{},[21,634,635],{},"publicationName",[37,637,638,641],{},[21,639,640],{},"initialSnapshot"," for copying existing rows before CDC starts",[14,643,644,43,646,43,648,50,650,652],{},[21,645,623],{},[21,647,629],{},[21,649,635],{},[21,651,640],{}," are CDC-only settings.",[14,654,655,656,235],{},"For initial snapshot behavior and target requirements, see ",[269,657,659],{"href":658},"\u002Fdocs\u002Fstreams\u002Finitial-load-and-cdc","Initial Load + CDC",[26,661,663],{"id":662},"verification","Verification",[14,665,666],{},"Before saving or using a source config:",[34,668,669,674,677,685,694],{},[37,670,671,672],{},"confirm every entry lives under ",[21,673,23],{},[37,675,676],{},"confirm each connection entry uses only one source family",[37,678,679,680,71,682,684],{},"confirm database sources use ",[21,681,49],{},[21,683,53],{},", not file-specific fields",[37,686,687,688,488,690,71,692],{},"confirm S3 sources do not mix ",[21,689,481],{},[21,691,491],{},[21,693,494],{},[37,695,696,697],{},"confirm local file paths are correct relative to ",[21,698,585],{},[26,700,702],{"id":701},"related-docs","Related docs",[34,704,705,711,717,723,729],{},[37,706,707],{},[269,708,710],{"href":709},"\u002Fdocs\u002Fstreams\u002Fstream-configuration-overview","Stream Configuration Overview",[37,712,713],{},[269,714,716],{"href":715},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Stream Configuration Reference",[37,718,719],{},[269,720,722],{"href":721},"\u002Fdocs\u002Fstreams\u002Fstream-examples","Stream Configuration Examples",[37,724,725],{},[269,726,728],{"href":727},"\u002Fdocs\u002Fstreams\u002Fschema-selection-guide","Schema Selection Guide",[37,730,731],{},[269,732,734],{"href":733},"\u002Fdocs\u002Fstreams\u002Fcustom-sql-queries","Custom SQL Queries",[736,737,738],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":91,"searchDepth":105,"depth":105,"links":740},[741,742,743,744,745,746,747,748,749],{"id":28,"depth":105,"text":29},{"id":77,"depth":105,"text":78},{"id":223,"depth":105,"text":224},{"id":275,"depth":105,"text":276},{"id":386,"depth":105,"text":387},{"id":497,"depth":105,"text":498},{"id":600,"depth":105,"text":601},{"id":662,"depth":105,"text":663},{"id":701,"depth":105,"text":702},"Define what a DBConvert Streams configuration reads from, including database, S3, and local file sources.","md",{},false,"\u002Fdocs\u002Fstreams\u002Fsource-configuration",null,{"title":5,"description":750},"docs\u002Fstreams\u002Fsource-configuration","w9OHzd0SaoLbUeKdJIv7MOIMXIE0YA4gN8MCXFlJN0o",1783896284828]