[{"data":1,"prerenderedAt":583},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fconnections\u002Fcompressed-files":3},{"id":4,"title":5,"body":6,"description":574,"extension":575,"meta":576,"navigation":577,"path":578,"redirect":579,"seo":580,"stem":581,"__hash__":582},"docs\u002Fdocs\u002Fconnections\u002Fcompressed-files.md","Compressed File Support",{"type":7,"value":8,"toc":561},"minimark",[9,13,17,34,39,42,47,110,113,143,157,178,182,185,188,192,195,229,232,236,247,251,258,262,318,401,405,411,471,526,530,557],[10,11,5],"h1",{"id":12},"compressed-file-support",[14,15,16],"p",{},"DBConvert Streams handles compression on both sides of a stream:",[18,19,20,28],"ul",{},[21,22,23,27],"li",{},[24,25,26],"strong",{},"Reading"," — compressed source files are decompressed transparently with no configuration needed",[21,29,30,33],{},[24,31,32],{},"Writing"," — file and S3 targets support configurable output compression",[35,36,38],"h2",{"id":37},"reading-compressed-files","Reading compressed files",[14,40,41],{},"The system detects, decompresses, and processes files using streaming techniques that keep memory usage low regardless of file size.",[43,44,46],"h3",{"id":45},"supported-read-formats","Supported read formats",[48,49,50,66],"table",{},[51,52,53],"thead",{},[54,55,56,60,63],"tr",{},[57,58,59],"th",{},"Compression",[57,61,62],{},"Extension",[57,64,65],{},"Typical ratio",[67,68,69,84,97],"tbody",{},[54,70,71,75,81],{},[72,73,74],"td",{},"Gzip",[72,76,77],{},[78,79,80],"code",{},".gz",[72,82,83],{},"5-10x for CSV, 3-8x for JSON",[54,85,86,89,94],{},[72,87,88],{},"Zstandard",[72,90,91],{},[78,92,93],{},".zst",[72,95,96],{},"Best balance of ratio and speed",[54,98,99,102,107],{},[72,100,101],{},"Bzip2",[72,103,104],{},[78,105,106],{},".bz2",[72,108,109],{},"Highest ratio, slower decompression",[14,111,112],{},"Works with CSV, JSON, and JSONL files:",[18,114,115,127,135],{},[21,116,117,120,121,120,124],{},[78,118,119],{},"data.csv.gz",", ",[78,122,123],{},"export.csv.bz2",[78,125,126],{},"logs.csv.zst",[21,128,129,120,132],{},[78,130,131],{},"logs.json.gz",[78,133,134],{},"backup.json.bz2",[21,136,137,120,140],{},[78,138,139],{},"stream.jsonl.gz",[78,141,142],{},"events.jsonl.zst",[144,145,148],"alert",{"title":146,"type":147},"Parquet handles compression differently","tip",[14,149,150,151,153,154,156],{},"Parquet files use built-in compression (Snappy, Zstd, Gzip) at the column-chunk level — they are not wrapped in ",[78,152,80],{}," or ",[78,155,93],{},". DBConvert Streams reads Parquet compression natively.",[14,158,159,160,120,162,164,165,167,168,153,171,174,175,177],{},"Compression is recognized from the file suffix, such as ",[78,161,80],{},[78,163,106],{},", or ",[78,166,93],{},". DBConvert Streams then uses the remaining extension, such as ",[78,169,170],{},".csv",[78,172,173],{},".jsonl",", to choose the correct reader. For example, ",[78,176,119],{}," is treated as a gzip-compressed CSV file.",[35,179,181],{"id":180},"processing","Processing",[14,183,184],{},"Compressed files are read through the normal file-reading path. DBConvert Streams identifies the compression and file format from the path and uses the matching reader.",[14,186,187],{},"For local files, the application does not create intermediate uncompressed files on disk while reading compressed inputs.",[35,189,191],{"id":190},"using-compressed-files","Using compressed files",[14,193,194],{},"Compressed files work transparently with existing workflows:",[18,196,197,203,209,215],{},[21,198,199,202],{},[24,200,201],{},"Data Explorer"," — browse and preview compressed files the same way as uncompressed ones",[21,204,205,208],{},[24,206,207],{},"SQL Console"," — query compressed files directly",[21,210,211,214],{},[24,212,213],{},"Stream sources"," — use compressed files as stream input",[21,216,217,220,221,224,225,228],{},[24,218,219],{},"API"," — the ",[78,222,223],{},"\u002Fapi\u002Fv1\u002Ffiles\u002Fdata"," and ",[78,226,227],{},"\u002Fapi\u002Fv1\u002Ffiles\u002Fmeta"," endpoints handle compressed files automatically",[14,230,231],{},"No special configuration is needed. Point to a compressed file path and the system handles the rest.",[35,233,235],{"id":234},"error-handling","Error handling",[18,237,238,241,244],{},[21,239,240],{},"If compression format cannot be determined, the file is treated as uncompressed",[21,242,243],{},"Corrupted compressed files produce clear error messages distinguishing compression errors from data format errors",[21,245,246],{},"Built-in safeguards prevent memory overflow from malformed data",[35,248,250],{"id":249},"writing-compressed-output","Writing compressed output",[14,252,253,254,257],{},"File and S3 stream targets support output compression via ",[78,255,256],{},"format.compression"," in the target spec.",[43,259,261],{"id":260},"csv-and-jsonl-targets","CSV and JSONL targets",[48,263,264,276],{},[51,265,266],{},[54,267,268,270,273],{},[57,269,59],{},[57,271,272],{},"Value",[57,274,275],{},"Notes",[67,277,278,291,303],{},[54,279,280,283,288],{},[72,281,282],{},"None",[72,284,285],{},[78,286,287],{},"uncompressed",[72,289,290],{},"Default",[54,292,293,295,300],{},[72,294,74],{},[72,296,297],{},[78,298,299],{},"gzip",[72,301,302],{},"Wide compatibility",[54,304,305,307,312],{},[72,306,88],{},[72,308,309],{},[78,310,311],{},"zstd",[72,313,314,315,317],{},"Smaller files with ",[78,316,93],{}," output",[319,320,325],"pre",{"className":321,"code":322,"language":323,"meta":324,"style":324},"language-json shiki shiki-themes github-light github-dark","\"spec\": {\n  \"files\": {\n    \"fileFormat\": \"csv\",\n    \"format\": {\n      \"compression\": \"gzip\"\n    }\n  }\n}\n","json","",[78,326,327,340,349,364,372,383,389,395],{"__ignoreMap":324},[328,329,332,336],"span",{"class":330,"line":331},"line",1,[328,333,335],{"class":334},"sZZnC","\"spec\"",[328,337,339],{"class":338},"sVt8B",": {\n",[328,341,343,347],{"class":330,"line":342},2,[328,344,346],{"class":345},"sj4cs","  \"files\"",[328,348,339],{"class":338},[328,350,352,355,358,361],{"class":330,"line":351},3,[328,353,354],{"class":345},"    \"fileFormat\"",[328,356,357],{"class":338},": ",[328,359,360],{"class":334},"\"csv\"",[328,362,363],{"class":338},",\n",[328,365,367,370],{"class":330,"line":366},4,[328,368,369],{"class":345},"    \"format\"",[328,371,339],{"class":338},[328,373,375,378,380],{"class":330,"line":374},5,[328,376,377],{"class":345},"      \"compression\"",[328,379,357],{"class":338},[328,381,382],{"class":334},"\"gzip\"\n",[328,384,386],{"class":330,"line":385},6,[328,387,388],{"class":338},"    }\n",[328,390,392],{"class":330,"line":391},7,[328,393,394],{"class":338},"  }\n",[328,396,398],{"class":330,"line":397},8,[328,399,400],{"class":338},"}\n",[43,402,404],{"id":403},"parquet-targets","Parquet targets",[14,406,407,408,410],{},"Parquet compression is controlled with ",[78,409,256],{},":",[48,412,413,423],{},[51,414,415],{},[54,416,417,419,421],{},[57,418,59],{},[57,420,272],{},[57,422,275],{},[67,424,425,436,449,460],{},[54,426,427,429,433],{},[72,428,88],{},[72,430,431],{},[78,432,311],{},[72,434,435],{},"Default — best balance of ratio and speed",[54,437,438,441,446],{},[72,439,440],{},"Snappy",[72,442,443],{},[78,444,445],{},"snappy",[72,447,448],{},"Fastest compression\u002Fdecompression",[54,450,451,453,457],{},[72,452,74],{},[72,454,455],{},[78,456,299],{},[72,458,459],{},"Highest ratio, slower",[54,461,462,464,468],{},[72,463,282],{},[72,465,466],{},[78,467,287],{},[72,469,470],{},"No compression",[319,472,474],{"className":321,"code":473,"language":323,"meta":324,"style":324},"\"spec\": {\n  \"files\": {\n    \"fileFormat\": \"parquet\",\n    \"format\": {\n      \"compression\": \"snappy\"\n    }\n  }\n}\n",[78,475,476,482,488,499,505,514,518,522],{"__ignoreMap":324},[328,477,478,480],{"class":330,"line":331},[328,479,335],{"class":334},[328,481,339],{"class":338},[328,483,484,486],{"class":330,"line":342},[328,485,346],{"class":345},[328,487,339],{"class":338},[328,489,490,492,494,497],{"class":330,"line":351},[328,491,354],{"class":345},[328,493,357],{"class":338},[328,495,496],{"class":334},"\"parquet\"",[328,498,363],{"class":338},[328,500,501,503],{"class":330,"line":366},[328,502,369],{"class":345},[328,504,339],{"class":338},[328,506,507,509,511],{"class":330,"line":374},[328,508,377],{"class":345},[328,510,357],{"class":338},[328,512,513],{"class":334},"\"snappy\"\n",[328,515,516],{"class":330,"line":385},[328,517,388],{"class":338},[328,519,520],{"class":330,"line":391},[328,521,394],{"class":338},[328,523,524],{"class":330,"line":397},[328,525,400],{"class":338},[35,527,529],{"id":528},"related-docs","Related docs",[18,531,532,539,545,551],{},[21,533,534],{},[535,536,538],"a",{"href":537},"\u002Fdocs\u002Fconnections\u002Flocal-files","Local Files",[21,540,541],{},[535,542,544],{"href":543},"\u002Fdocs\u002Fconnections\u002Fs3-compatible-storage","S3-Compatible Storage",[21,546,547],{},[535,548,550],{"href":549},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Stream Configuration Reference",[21,552,553],{},[535,554,556],{"href":555},"\u002Fdocs\u002Fapi\u002Ffiles-and-storage","Files & Storage API Workflows",[558,559,560],"style",{},"html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":324,"searchDepth":342,"depth":342,"links":562},[563,566,567,568,569,573],{"id":37,"depth":342,"text":38,"children":564},[565],{"id":45,"depth":351,"text":46},{"id":180,"depth":342,"text":181},{"id":190,"depth":342,"text":191},{"id":234,"depth":342,"text":235},{"id":249,"depth":342,"text":250,"children":570},[571,572],{"id":260,"depth":351,"text":261},{"id":403,"depth":351,"text":404},{"id":528,"depth":342,"text":529},"Read compressed source files and configure compression for file and S3 stream targets in DBConvert Streams.","md",{},false,"\u002Fdocs\u002Fconnections\u002Fcompressed-files",null,{"title":5,"description":574},"docs\u002Fconnections\u002Fcompressed-files","ELGwW8sTSV5YkqklhkzvtZLn1yRP7VWnpDR_TuIdyb8",1783896284379]