[{"data":1,"prerenderedAt":721},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fstreams\u002Fcustom-sql-queries":3},{"id":4,"title":5,"body":6,"description":712,"extension":713,"meta":714,"navigation":715,"path":716,"redirect":717,"seo":718,"stem":719,"__hash__":720},"docs\u002Fdocs\u002Fstreams\u002Fcustom-sql-queries.md","Custom SQL Queries",{"type":7,"value":8,"toc":703},"minimark",[9,13,28,41,46,53,76,79,83,316,327,331,362,366,376,388,391,541,549,553,632,638,642,673,677,699],[10,11,5],"h1",{"id":12},"custom-sql-queries",[14,15,16,17,21,22,27],"p",{},"Custom SQL queries let you define free-form ",[18,19,20],"code",{},"SELECT"," statements that produce virtual tables in the target. Unlike ",[23,24,26],"a",{"href":25},"\u002Fdocs\u002Fstreams\u002Fdata-filter-guide","structured data filters"," (which modify how existing tables are read), queries create entirely new result sets from arbitrary SQL — including JOINs, CTEs, aggregations, and expressions.",[14,29,30,31,35,36,40],{},"Available in ",[32,33,34],"strong",{},"Load mode only",". For cross-source SQL, use ",[23,37,39],{"href":38},"\u002Fdocs\u002Fdatabase-explorer\u002Ffederated-queries","Federated Queries",".",[42,43,45],"h2",{"id":44},"how-queries-work","How queries work",[14,47,48,49,52],{},"Each query entry in the ",[18,50,51],{},"queries"," array defines:",[54,55,56,65],"ul",{},[57,58,59,64],"li",{},[32,60,61],{},[18,62,63],{},"name"," — the virtual table name created in the target",[57,66,67,72,73,75],{},[32,68,69],{},[18,70,71],{},"query"," — a full ",[18,74,20],{}," statement executed against the source",[14,77,78],{},"The backend executes queries via DuckDB, writing the result set to the target under the given name.",[42,80,82],{"id":81},"example","Example",[84,85,90],"pre",{"className":86,"code":87,"language":88,"meta":89,"style":89},"language-json shiki shiki-themes github-light github-dark","{\n  \"name\": \"users-transform\",\n  \"mode\": \"load\",\n  \"source\": {\n    \"connections\": [\n      {\n        \"connectionId\": \"conn_SOURCE_ID\",\n        \"database\": \"app_db\",\n        \"queries\": [\n          {\n            \"name\": \"active_customers\",\n            \"query\": \"SELECT id, UPPER(full_name) AS full_name, LOWER(email) AS email FROM customers WHERE active = true\"\n          }\n        ]\n      }\n    ]\n  },\n  \"target\": {\n    \"id\": \"conn_TARGET_ID\",\n    \"spec\": {\n      \"db\": { \"database\": \"warehouse\" }\n    }\n  }\n}\n","json","",[18,91,92,101,118,131,140,149,155,168,181,189,195,208,219,225,231,237,243,249,257,270,278,298,304,310],{"__ignoreMap":89},[93,94,97],"span",{"class":95,"line":96},"line",1,[93,98,100],{"class":99},"sVt8B","{\n",[93,102,104,108,111,115],{"class":95,"line":103},2,[93,105,107],{"class":106},"sj4cs","  \"name\"",[93,109,110],{"class":99},": ",[93,112,114],{"class":113},"sZZnC","\"users-transform\"",[93,116,117],{"class":99},",\n",[93,119,121,124,126,129],{"class":95,"line":120},3,[93,122,123],{"class":106},"  \"mode\"",[93,125,110],{"class":99},[93,127,128],{"class":113},"\"load\"",[93,130,117],{"class":99},[93,132,134,137],{"class":95,"line":133},4,[93,135,136],{"class":106},"  \"source\"",[93,138,139],{"class":99},": {\n",[93,141,143,146],{"class":95,"line":142},5,[93,144,145],{"class":106},"    \"connections\"",[93,147,148],{"class":99},": [\n",[93,150,152],{"class":95,"line":151},6,[93,153,154],{"class":99},"      {\n",[93,156,158,161,163,166],{"class":95,"line":157},7,[93,159,160],{"class":106},"        \"connectionId\"",[93,162,110],{"class":99},[93,164,165],{"class":113},"\"conn_SOURCE_ID\"",[93,167,117],{"class":99},[93,169,171,174,176,179],{"class":95,"line":170},8,[93,172,173],{"class":106},"        \"database\"",[93,175,110],{"class":99},[93,177,178],{"class":113},"\"app_db\"",[93,180,117],{"class":99},[93,182,184,187],{"class":95,"line":183},9,[93,185,186],{"class":106},"        \"queries\"",[93,188,148],{"class":99},[93,190,192],{"class":95,"line":191},10,[93,193,194],{"class":99},"          {\n",[93,196,198,201,203,206],{"class":95,"line":197},11,[93,199,200],{"class":106},"            \"name\"",[93,202,110],{"class":99},[93,204,205],{"class":113},"\"active_customers\"",[93,207,117],{"class":99},[93,209,211,214,216],{"class":95,"line":210},12,[93,212,213],{"class":106},"            \"query\"",[93,215,110],{"class":99},[93,217,218],{"class":113},"\"SELECT id, UPPER(full_name) AS full_name, LOWER(email) AS email FROM customers WHERE active = true\"\n",[93,220,222],{"class":95,"line":221},13,[93,223,224],{"class":99},"          }\n",[93,226,228],{"class":95,"line":227},14,[93,229,230],{"class":99},"        ]\n",[93,232,234],{"class":95,"line":233},15,[93,235,236],{"class":99},"      }\n",[93,238,240],{"class":95,"line":239},16,[93,241,242],{"class":99},"    ]\n",[93,244,246],{"class":95,"line":245},17,[93,247,248],{"class":99},"  },\n",[93,250,252,255],{"class":95,"line":251},18,[93,253,254],{"class":106},"  \"target\"",[93,256,139],{"class":99},[93,258,260,263,265,268],{"class":95,"line":259},19,[93,261,262],{"class":106},"    \"id\"",[93,264,110],{"class":99},[93,266,267],{"class":113},"\"conn_TARGET_ID\"",[93,269,117],{"class":99},[93,271,273,276],{"class":95,"line":272},20,[93,274,275],{"class":106},"    \"spec\"",[93,277,139],{"class":99},[93,279,281,284,287,290,292,295],{"class":95,"line":280},21,[93,282,283],{"class":106},"      \"db\"",[93,285,286],{"class":99},": { ",[93,288,289],{"class":106},"\"database\"",[93,291,110],{"class":99},[93,293,294],{"class":113},"\"warehouse\"",[93,296,297],{"class":99}," }\n",[93,299,301],{"class":95,"line":300},22,[93,302,303],{"class":99},"    }\n",[93,305,307],{"class":95,"line":306},23,[93,308,309],{"class":99},"  }\n",[93,311,313],{"class":95,"line":312},24,[93,314,315],{"class":99},"}\n",[14,317,318,319,322,323,326],{},"This creates a ",[18,320,321],{},"active_customers"," table in the target ",[18,324,325],{},"warehouse"," database with the filtered and transformed result.",[42,328,330],{"id":329},"practical-rules","Practical rules",[54,332,333,339,345,348,351,354],{},[57,334,335,336,338],{},"Use ",[18,337,20],{}," statements only.",[57,340,341,342,344],{},"The ",[18,343,63],{}," field becomes the target table name — keep it stable across runs.",[57,346,347],{},"Return only the columns the target needs.",[57,349,350],{},"If the target depends on primary keys, include them in the result.",[57,352,353],{},"Keep column aliases stable so the resulting schema is predictable.",[57,355,356,357,361],{},"Validate the query in the ",[23,358,360],{"href":359},"\u002Fdocs\u002Fdatabase-explorer\u002Fsql-console","SQL Console"," before running a full stream.",[42,363,365],{"id":364},"mixing-tables-and-queries-in-one-stream","Mixing tables and queries in one stream",[14,367,368,369,372,373,375],{},"A single database connection can include both ",[18,370,371],{},"tables"," and ",[18,374,51],{}," in the same stream. The backend runs this as a mixed stream:",[377,378,379,382,385],"ol",{},[57,380,381],{},"Tables are streamed in parallel using native database readers.",[57,383,384],{},"Custom queries are executed sequentially via DuckDB.",[57,386,387],{},"Results from both phases are aggregated into the target.",[14,389,390],{},"This lets you combine straightforward table copies with transformed query extracts in a single Load run.",[84,392,394],{"className":86,"code":393,"language":88,"meta":89,"style":89},"{\n  \"mode\": \"load\",\n  \"source\": {\n    \"connections\": [\n      {\n        \"connectionId\": \"conn_PG_ID\",\n        \"database\": \"app_db\",\n        \"tables\": [\n          { \"name\": \"orders\" },\n          { \"name\": \"products\" }\n        ],\n        \"queries\": [\n          {\n            \"name\": \"active_customers\",\n            \"query\": \"SELECT id, email, plan FROM customers WHERE active = true\"\n          }\n        ]\n      }\n    ]\n  }\n}\n",[18,395,396,400,410,416,422,426,437,447,454,470,483,488,494,498,508,517,521,525,529,533,537],{"__ignoreMap":89},[93,397,398],{"class":95,"line":96},[93,399,100],{"class":99},[93,401,402,404,406,408],{"class":95,"line":103},[93,403,123],{"class":106},[93,405,110],{"class":99},[93,407,128],{"class":113},[93,409,117],{"class":99},[93,411,412,414],{"class":95,"line":120},[93,413,136],{"class":106},[93,415,139],{"class":99},[93,417,418,420],{"class":95,"line":133},[93,419,145],{"class":106},[93,421,148],{"class":99},[93,423,424],{"class":95,"line":142},[93,425,154],{"class":99},[93,427,428,430,432,435],{"class":95,"line":151},[93,429,160],{"class":106},[93,431,110],{"class":99},[93,433,434],{"class":113},"\"conn_PG_ID\"",[93,436,117],{"class":99},[93,438,439,441,443,445],{"class":95,"line":157},[93,440,173],{"class":106},[93,442,110],{"class":99},[93,444,178],{"class":113},[93,446,117],{"class":99},[93,448,449,452],{"class":95,"line":170},[93,450,451],{"class":106},"        \"tables\"",[93,453,148],{"class":99},[93,455,456,459,462,464,467],{"class":95,"line":183},[93,457,458],{"class":99},"          { ",[93,460,461],{"class":106},"\"name\"",[93,463,110],{"class":99},[93,465,466],{"class":113},"\"orders\"",[93,468,469],{"class":99}," },\n",[93,471,472,474,476,478,481],{"class":95,"line":191},[93,473,458],{"class":99},[93,475,461],{"class":106},[93,477,110],{"class":99},[93,479,480],{"class":113},"\"products\"",[93,482,297],{"class":99},[93,484,485],{"class":95,"line":197},[93,486,487],{"class":99},"        ],\n",[93,489,490,492],{"class":95,"line":210},[93,491,186],{"class":106},[93,493,148],{"class":99},[93,495,496],{"class":95,"line":221},[93,497,194],{"class":99},[93,499,500,502,504,506],{"class":95,"line":227},[93,501,200],{"class":106},[93,503,110],{"class":99},[93,505,205],{"class":113},[93,507,117],{"class":99},[93,509,510,512,514],{"class":95,"line":233},[93,511,213],{"class":106},[93,513,110],{"class":99},[93,515,516],{"class":113},"\"SELECT id, email, plan FROM customers WHERE active = true\"\n",[93,518,519],{"class":95,"line":239},[93,520,224],{"class":99},[93,522,523],{"class":95,"line":245},[93,524,230],{"class":99},[93,526,527],{"class":95,"line":251},[93,528,236],{"class":99},[93,530,531],{"class":95,"line":259},[93,532,242],{"class":99},[93,534,535],{"class":95,"line":272},[93,536,309],{"class":99},[93,538,539],{"class":95,"line":280},[93,540,315],{"class":99},[14,542,543,544,548],{},"In ",[23,545,547],{"href":546},"\u002Fdocs\u002Fstreams\u002Fmulti-source-streams","multi-source streams",", each connection defines its own tables and queries independently. You can also mix connection types — for example, a PostgreSQL connection with tables + queries alongside an S3 connection with file selections.",[42,550,552],{"id":551},"queries-vs-data-filters","Queries vs data filters",[554,555,556,579],"table",{},[557,558,559],"thead",{},[560,561,562,566,573],"tr",{},[563,564,565],"th",{},"Feature",[563,567,568,569,572],{},"Custom SQL Queries (",[18,570,571],{},"queries[]",")",[563,574,575,576,572],{},"Data Filters (",[18,577,578],{},"tables[].selection",[580,581,582,597,611,622],"tbody",{},[560,583,584,588,594],{},[585,586,587],"td",{},"Input",[585,589,590,591,593],{},"Free-form ",[18,592,20],{}," statement",[585,595,596],{},"Visual column\u002Frow\u002Fsort\u002Flimit controls",[560,598,599,602,608],{},[585,600,601],{},"Target table name",[585,603,604,605,607],{},"Defined by ",[18,606,63],{}," field",[585,609,610],{},"Same as source table name",[560,612,613,616,619],{},[585,614,615],{},"JOINs, CTEs, aggregations",[585,617,618],{},"Yes",[585,620,621],{},"No",[560,623,624,627,630],{},[585,625,626],{},"CDC support",[585,628,629],{},"No (Load only)",[585,631,629],{},[14,633,634,635,40],{},"For simple column selection and row filtering without writing SQL, use ",[23,636,637],{"href":25},"Data Filters",[42,639,641],{"id":640},"validation-checklist","Validation checklist",[377,643,644,651,656,659,666],{},[57,645,646,647,40],{},"Verify the source connection in ",[23,648,650],{"href":649},"\u002Fdocs\u002Fdatabase-explorer\u002F","Data Explorer",[57,652,653,654,40],{},"Run the query in the ",[23,655,360],{"href":359},[57,657,658],{},"Confirm the column names and types match the target expectation.",[57,660,661,662,665],{},"Start with a limited run (",[18,663,664],{},"limits.numberOfEvents",") before applying broadly.",[57,667,668,669,672],{},"After the first run, verify the stream reached ",[32,670,671],{},"finished"," and the target table has the expected rows.",[42,674,676],{"id":675},"related-docs","Related docs",[54,678,679,683,689,693],{},[57,680,681],{},[23,682,637],{"href":25},[57,684,685],{},[23,686,688],{"href":687},"\u002Fdocs\u002Fstreams\u002Fschema-selection-guide","Schema Selection Guide",[57,690,691],{},[23,692,39],{"href":38},[57,694,695],{},[23,696,698],{"href":697},"\u002Fdocs\u002Fstreams\u002Fstream-configuration-guide","Stream Configuration Guide",[700,701,702],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":89,"searchDepth":103,"depth":103,"links":704},[705,706,707,708,709,710,711],{"id":44,"depth":103,"text":45},{"id":81,"depth":103,"text":82},{"id":329,"depth":103,"text":330},{"id":364,"depth":103,"text":365},{"id":551,"depth":103,"text":552},{"id":640,"depth":103,"text":641},{"id":675,"depth":103,"text":676},"Use custom SELECT queries to narrow or reshape table reads in DBConvert Streams Load workflows.","md",{},false,"\u002Fdocs\u002Fstreams\u002Fcustom-sql-queries",null,{"title":5,"description":712},"docs\u002Fstreams\u002Fcustom-sql-queries","SmYIGxGf7jzVLDBjPFQx39SzZmNR8o8XerBZEvGLrvE",1783896284142]