[{"data":1,"prerenderedAt":977},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fstreams\u002Fdata-filter-guide":3},{"id":4,"title":5,"body":6,"description":968,"extension":969,"meta":970,"navigation":971,"path":972,"redirect":973,"seo":974,"stem":975,"__hash__":976},"docs\u002Fdocs\u002Fstreams\u002Fdata-filter-guide.md","Data filter",{"type":7,"value":8,"toc":951},"minimark",[9,13,17,20,36,45,50,67,89,93,99,117,125,129,135,179,188,193,375,394,398,424,428,433,454,458,464,483,486,491,512,515,519,526,567,570,574,577,784,798,808,812,901,908,912,919,923,947],[10,11,5],"h1",{"id":12},"data-filter",[14,15,16],"p",{},"The Data Filter panel lets you control exactly which data leaves the source during a Load workflow. Instead of writing SQL by hand, you configure column selection, row filters, sort order, and row limits through a visual interface.",[14,18,19],{},"Use this when:",[21,22,23,27,30,33],"ul",{},[24,25,26],"li",{},"you want a subset of columns instead of the full table",[24,28,29],{},"you need to exclude rows based on conditions (e.g. status, date range, null values)",[24,31,32],{},"you want rows sorted in a specific order at the source",[24,34,35],{},"you want to cap the number of rows transferred",[14,37,38,39,44],{},"For full SQL control (transformations, aliases, expressions), use ",[40,41,43],"a",{"href":42},"\u002Fdocs\u002Fstreams\u002Fcustom-sql-queries","Custom SQL Queries"," instead.",[46,47,49],"h2",{"id":48},"opening-the-data-filter","Opening the Data Filter",[51,52,53,61,64],"ol",{},[24,54,55,56,60],{},"In the stream configuration wizard, reach ",[57,58,59],"strong",{},"Step 2: Structure and Data",".",[24,62,63],{},"Select a table by clicking its checkbox.",[24,65,66],{},"Click the filter icon next to the table name.",[14,68,69,70,73,74,73,77,80,81,84,85,88],{},"The Data Filter panel opens on the right with four sections accessible from the header: ",[57,71,72],{},"Columns",", ",[57,75,76],{},"Filter",[57,78,79],{},"Sort",", and a ",[57,82,83],{},"Limit"," field. A ",[57,86,87],{},"Preview"," button validates your configuration against live data.",[46,90,92],{"id":91},"column-selection","Column selection",[14,94,95,96,98],{},"Click ",[57,97,72],{}," in the header to open the column selector.",[21,100,101,111,114],{},[24,102,103,106,107,110],{},[57,104,105],{},"All"," \u002F ",[57,108,109],{},"None"," buttons toggle the full set.",[24,112,113],{},"Click individual column chips to include or exclude them.",[24,115,116],{},"When a subset is selected, the Columns button shows a badge with the count.",[14,118,119,120,124],{},"If no columns are explicitly selected (or all are selected), the query uses ",[121,122,123],"code",{},"SELECT *",". Otherwise, only the selected columns are read from the source and written to the target.",[46,126,128],{"id":127},"row-filters","Row filters",[14,130,95,131,134],{},[57,132,133],{},"+ Filter"," to add a WHERE condition. Each filter row has three parts:",[136,137,138,151],"table",{},[139,140,141],"thead",{},[142,143,144,148],"tr",{},[145,146,147],"th",{},"Part",[145,149,150],{},"Description",[152,153,154,163,171],"tbody",{},[142,155,156,160],{},[157,158,159],"td",{},"Column",[157,161,162],{},"The source column to test",[142,164,165,168],{},[157,166,167],{},"Operator",[157,169,170],{},"The comparison to apply",[142,172,173,176],{},[157,174,175],{},"Value",[157,177,178],{},"The value to compare against (hidden for IS NULL \u002F IS NOT NULL)",[14,180,181,182,185,186,44],{},"Multiple filter rows are combined with ",[57,183,184],{},"AND",". Only rows matching all conditions are transferred. OR logic is not supported — if you need OR conditions, use ",[40,187,43],{"href":42},[189,190,192],"h3",{"id":191},"supported-operators","Supported operators",[136,194,195,207],{},[139,196,197],{},[142,198,199,201,204],{},[145,200,167],{},[145,202,203],{},"Value format",[145,205,206],{},"Example SQL",[152,208,209,224,238,261,276,290,304,318,333,347,361],{},[142,210,211,216,219],{},[157,212,213],{},[121,214,215],{},"=",[157,217,218],{},"Single value",[157,220,221],{},[121,222,223],{},"col = 'value'",[142,225,226,231,233],{},[157,227,228],{},[121,229,230],{},"!=",[157,232,218],{},[157,234,235],{},[121,236,237],{},"col != 'value'",[142,239,240,254,256],{},[157,241,242,73,245,73,248,73,251],{},[121,243,244],{},"\u003C",[121,246,247],{},"\u003C=",[121,249,250],{},">",[121,252,253],{},">=",[157,255,218],{},[157,257,258],{},[121,259,260],{},"col >= 100",[142,262,263,268,271],{},[157,264,265],{},[121,266,267],{},"CONTAINS",[157,269,270],{},"Text",[157,272,273],{},[121,274,275],{},"col LIKE '%value%'",[142,277,278,283,285],{},[157,279,280],{},[121,281,282],{},"NOT CONTAINS",[157,284,270],{},[157,286,287],{},[121,288,289],{},"col NOT LIKE '%value%'",[142,291,292,297,299],{},[157,293,294],{},[121,295,296],{},"STARTS WITH",[157,298,270],{},[157,300,301],{},[121,302,303],{},"col LIKE 'value%'",[142,305,306,311,313],{},[157,307,308],{},[121,309,310],{},"ENDS WITH",[157,312,270],{},[157,314,315],{},[121,316,317],{},"col LIKE '%value'",[142,319,320,325,328],{},[157,321,322],{},[121,323,324],{},"IN",[157,326,327],{},"Comma-separated",[157,329,330],{},[121,331,332],{},"col IN ('a', 'b', 'c')",[142,334,335,340,342],{},[157,336,337],{},[121,338,339],{},"NOT IN",[157,341,327],{},[157,343,344],{},[121,345,346],{},"col NOT IN ('a', 'b')",[142,348,349,354,356],{},[157,350,351],{},[121,352,353],{},"IS NULL",[157,355,109],{},[157,357,358],{},[121,359,360],{},"col IS NULL",[142,362,363,368,370],{},[157,364,365],{},[121,366,367],{},"IS NOT NULL",[157,369,109],{},[157,371,372],{},[121,373,374],{},"col IS NOT NULL",[376,377,380],"alert",{"title":378,"type":379},"PostgreSQL dialect","info",[14,381,382,383,73,385,73,387,389,390,393],{},"Pattern operators (",[121,384,267],{},[121,386,296],{},[121,388,310],{},") use ",[121,391,392],{},"ILIKE"," on PostgreSQL for case-insensitive matching.",[189,395,397],{"id":396},"filter-behavior","Filter behavior",[21,399,400,403,418,421],{},[24,401,402],{},"A filter row is applied when it has a column and a value (except for IS NULL \u002F IS NOT NULL).",[24,404,405,406,409,410,413,414,417],{},"A filtered column does not need to be included in ",[121,407,408],{},"selectedColumns",". Column selection controls the ",[121,411,412],{},"SELECT"," list; filters control the ",[121,415,416],{},"WHERE"," clause.",[24,419,420],{},"String values containing single quotes are escaped automatically.",[24,422,423],{},"IN \u002F NOT IN values are trimmed and split by comma.",[46,425,427],{"id":426},"sort-order","Sort order",[14,429,95,430,432],{},[57,431,79],{}," to add an ORDER BY clause.",[21,434,435,438,448,451],{},[24,436,437],{},"Choose a column from the dropdown.",[24,439,440,441,444,445,60],{},"Toggle the direction button: blue arrow up for ",[57,442,443],{},"ASC",", amber arrow down for ",[57,446,447],{},"DESC",[24,449,450],{},"Add multiple sort rows for multi-column ordering. The first row is the primary sort.",[24,452,453],{},"Each column can appear in only one sort row.",[46,455,457],{"id":456},"row-limit","Row limit",[14,459,460,461,463],{},"The ",[57,462,83],{}," field in the header controls how many rows are read.",[21,465,466,473,476],{},[24,467,468,469,472],{},"Leave empty or set to ",[121,470,471],{},"0"," for no limit (all rows).",[24,474,475],{},"Enter a positive number to cap the row count.",[24,477,478,479,482],{},"Click the ",[57,480,481],{},"X"," button to clear a previously set limit.",[46,484,87],{"id":485},"preview",[14,487,95,488,490],{},[57,489,87],{}," to run your filter configuration against the source and see matching rows.",[21,492,493,496,503,506,509],{},[24,494,495],{},"Preview returns up to 10 rows.",[24,497,498,499,502],{},"A row count badge shows how many rows matched (with a ",[121,500,501],{},"+"," indicator if more exist).",[24,504,505],{},"Only selected columns appear in the preview table.",[24,507,508],{},"If the filter produces an error, a red error panel shows the message.",[24,510,511],{},"If no rows match, an empty-state message is displayed.",[14,513,514],{},"Use Preview to validate your filters before starting the stream.",[46,516,518],{"id":517},"how-filters-translate-to-sql","How filters translate to SQL",[14,520,521,522,525],{},"The Data Filter panel builds a structured ",[121,523,524],{},"TableSelectionState"," that the backend converts to SQL at execution time. The generated query follows this pattern:",[527,528,533],"pre",{"className":529,"code":530,"language":531,"meta":532,"style":532},"language-sql shiki shiki-themes github-light github-dark","SELECT [selected columns or *]\nFROM tableName\nWHERE [filter1 AND filter2 AND ...]\nORDER BY [sort1, sort2, ...]\nLIMIT limit_value\n","sql","",[121,534,535,543,549,555,561],{"__ignoreMap":532},[536,537,540],"span",{"class":538,"line":539},"line",1,[536,541,542],{},"SELECT [selected columns or *]\n",[536,544,546],{"class":538,"line":545},2,[536,547,548],{},"FROM tableName\n",[536,550,552],{"class":538,"line":551},3,[536,553,554],{},"WHERE [filter1 AND filter2 AND ...]\n",[536,556,558],{"class":538,"line":557},4,[536,559,560],{},"ORDER BY [sort1, sort2, ...]\n",[536,562,564],{"class":538,"line":563},5,[536,565,566],{},"LIMIT limit_value\n",[14,568,569],{},"Column and table names are quoted according to the source dialect (backticks for MySQL, double quotes for PostgreSQL).",[189,571,573],{"id":572},"api-representation","API representation",[14,575,576],{},"In the stream configuration API, the selection state is stored as a structured object on each table entry:",[527,578,582],{"className":579,"code":580,"language":581,"meta":532,"style":532},"language-json shiki shiki-themes github-light github-dark","{\n  \"name\": \"film\",\n  \"selection\": {\n    \"selectedColumns\": [\"film_id\", \"title\", \"release_year\", \"rating\"],\n    \"filters\": [\n      { \"column\": \"release_year\", \"operator\": \">=\", \"value\": \"2000\" },\n      { \"column\": \"rating\", \"operator\": \"IN\", \"value\": \"PG, PG-13\" }\n    ],\n    \"sorts\": [\n      { \"column\": \"release_year\", \"direction\": \"DESC\" }\n    ],\n    \"limit\": 500\n  }\n}\n","json",[121,583,584,590,606,614,643,651,687,719,725,733,756,761,772,778],{"__ignoreMap":532},[536,585,586],{"class":538,"line":539},[536,587,589],{"class":588},"sVt8B","{\n",[536,591,592,596,599,603],{"class":538,"line":545},[536,593,595],{"class":594},"sj4cs","  \"name\"",[536,597,598],{"class":588},": ",[536,600,602],{"class":601},"sZZnC","\"film\"",[536,604,605],{"class":588},",\n",[536,607,608,611],{"class":538,"line":551},[536,609,610],{"class":594},"  \"selection\"",[536,612,613],{"class":588},": {\n",[536,615,616,619,622,625,627,630,632,635,637,640],{"class":538,"line":557},[536,617,618],{"class":594},"    \"selectedColumns\"",[536,620,621],{"class":588},": [",[536,623,624],{"class":601},"\"film_id\"",[536,626,73],{"class":588},[536,628,629],{"class":601},"\"title\"",[536,631,73],{"class":588},[536,633,634],{"class":601},"\"release_year\"",[536,636,73],{"class":588},[536,638,639],{"class":601},"\"rating\"",[536,641,642],{"class":588},"],\n",[536,644,645,648],{"class":538,"line":563},[536,646,647],{"class":594},"    \"filters\"",[536,649,650],{"class":588},": [\n",[536,652,654,657,660,662,664,666,669,671,674,676,679,681,684],{"class":538,"line":653},6,[536,655,656],{"class":588},"      { ",[536,658,659],{"class":594},"\"column\"",[536,661,598],{"class":588},[536,663,634],{"class":601},[536,665,73],{"class":588},[536,667,668],{"class":594},"\"operator\"",[536,670,598],{"class":588},[536,672,673],{"class":601},"\">=\"",[536,675,73],{"class":588},[536,677,678],{"class":594},"\"value\"",[536,680,598],{"class":588},[536,682,683],{"class":601},"\"2000\"",[536,685,686],{"class":588}," },\n",[536,688,690,692,694,696,698,700,702,704,707,709,711,713,716],{"class":538,"line":689},7,[536,691,656],{"class":588},[536,693,659],{"class":594},[536,695,598],{"class":588},[536,697,639],{"class":601},[536,699,73],{"class":588},[536,701,668],{"class":594},[536,703,598],{"class":588},[536,705,706],{"class":601},"\"IN\"",[536,708,73],{"class":588},[536,710,678],{"class":594},[536,712,598],{"class":588},[536,714,715],{"class":601},"\"PG, PG-13\"",[536,717,718],{"class":588}," }\n",[536,720,722],{"class":538,"line":721},8,[536,723,724],{"class":588},"    ],\n",[536,726,728,731],{"class":538,"line":727},9,[536,729,730],{"class":594},"    \"sorts\"",[536,732,650],{"class":588},[536,734,736,738,740,742,744,746,749,751,754],{"class":538,"line":735},10,[536,737,656],{"class":588},[536,739,659],{"class":594},[536,741,598],{"class":588},[536,743,634],{"class":601},[536,745,73],{"class":588},[536,747,748],{"class":594},"\"direction\"",[536,750,598],{"class":588},[536,752,753],{"class":601},"\"DESC\"",[536,755,718],{"class":588},[536,757,759],{"class":538,"line":758},11,[536,760,724],{"class":588},[536,762,764,767,769],{"class":538,"line":763},12,[536,765,766],{"class":594},"    \"limit\"",[536,768,598],{"class":588},[536,770,771],{"class":594},"500\n",[536,773,775],{"class":538,"line":774},13,[536,776,777],{"class":588},"  }\n",[536,779,781],{"class":538,"line":780},14,[536,782,783],{"class":588},"}\n",[14,785,786,787,73,790,793,794,797],{},"The saved payload uses API operator values such as ",[121,788,789],{},"NOT_CONTAINS",[121,791,792],{},"STARTS_WITH",", and ",[121,795,796],{},"ENDS_WITH",", even though the UI shows friendlier labels.",[14,799,800,801,804,805,60],{},"All fields are optional. Omitting ",[121,802,803],{},"selection"," entirely is equivalent to ",[121,806,807],{},"SELECT * FROM tableName",[46,809,811],{"id":810},"data-filter-vs-custom-sql-queries","Data filter vs custom SQL queries",[136,813,814,825],{},[139,815,816],{},[142,817,818,820,822],{},[145,819],{},[145,821,5],{},[145,823,824],{},"Custom SQL query",[152,826,827,838,848,859,870,880,890],{},[142,828,829,832,835],{},[157,830,831],{},"Interface",[157,833,834],{},"Visual panel",[157,836,837],{},"Raw SQL editor",[142,839,840,842,845],{},[157,841,92],{},[157,843,844],{},"Checkbox-based",[157,846,847],{},"Write SELECT list",[142,849,850,853,856],{},[157,851,852],{},"Row filtering",[157,854,855],{},"Operator dropdowns",[157,857,858],{},"Write WHERE clause",[142,860,861,864,867],{},[157,862,863],{},"Transformations",[157,865,866],{},"Not supported",[157,868,869],{},"Full SQL expressions",[142,871,872,875,877],{},[157,873,874],{},"Aliases",[157,876,866],{},[157,878,879],{},"Supported",[142,881,882,884,887],{},[157,883,87],{},[157,885,886],{},"Built-in",[157,888,889],{},"Use SQL Console",[142,891,892,895,898],{},[157,893,894],{},"Best for",[157,896,897],{},"Simple filtering without SQL knowledge",[157,899,900],{},"Complex transformations and expressions",[14,902,903,904,907],{},"Both approaches apply only to ",[57,905,906],{},"Load"," workflows.",[46,909,911],{"id":910},"verification","Verification",[14,913,914,915,918],{},"After running a filtered stream, check that the stream reached ",[57,916,917],{},"finished"," and the transferred row count matches what Preview showed. If the count is unexpectedly low or high, review your filter conditions and re-run Preview.",[46,920,922],{"id":921},"related-docs","Related docs",[21,924,925,929,935,941],{},[24,926,927],{},[40,928,43],{"href":42},[24,930,931],{},[40,932,934],{"href":933},"\u002Fdocs\u002Fstreams\u002Fschema-selection-guide","Schema Selection Guide",[24,936,937],{},[40,938,940],{"href":939},"\u002Fdocs\u002Fstreams\u002Fstream-configuration-guide","Stream Configuration Guide",[24,942,943],{},[40,944,946],{"href":945},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Configuration Reference",[948,949,950],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}",{"title":532,"searchDepth":545,"depth":545,"links":952},[953,954,955,959,960,961,962,965,966,967],{"id":48,"depth":545,"text":49},{"id":91,"depth":545,"text":92},{"id":127,"depth":545,"text":128,"children":956},[957,958],{"id":191,"depth":551,"text":192},{"id":396,"depth":551,"text":397},{"id":426,"depth":545,"text":427},{"id":456,"depth":545,"text":457},{"id":485,"depth":545,"text":87},{"id":517,"depth":545,"text":518,"children":963},[964],{"id":572,"depth":551,"text":573},{"id":810,"depth":545,"text":811},{"id":910,"depth":545,"text":911},{"id":921,"depth":545,"text":922},"Use the Data Filter panel to select columns, filter rows, sort results, and limit row counts before transferring data.","md",{},false,"\u002Fdocs\u002Fstreams\u002Fdata-filter-guide",null,{"title":5,"description":968},"docs\u002Fstreams\u002Fdata-filter-guide","j0JTslWrdyxsNcHPxVmgWjlDzrm9OjLWKTidiY0sfjc",1783896284916]