[{"data":1,"prerenderedAt":1733},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fconnections\u002Fs3-compatible-storage":3},{"id":4,"title":5,"body":6,"description":1724,"extension":1725,"meta":1726,"navigation":1727,"path":1728,"redirect":1729,"seo":1730,"stem":1731,"__hash__":1732},"docs\u002Fdocs\u002Fconnections\u002Fs3-compatible-storage.md","S3-Compatible Storage",{"type":7,"value":8,"toc":1701},"minimark",[9,13,17,22,70,74,197,203,237,248,252,255,416,425,434,443,452,456,522,530,534,565,569,574,604,608,611,631,636,940,1005,1009,1016,1024,1027,1031,1042,1046,1060,1152,1155,1159,1163,1170,1315,1483,1490,1494,1500,1503,1517,1521,1524,1528,1571,1574,1583,1587,1591,1594,1613,1621,1625,1639,1653,1657,1665,1669,1697],[10,11,5],"h1",{"id":12},"s3-compatible-storage",[14,15,16],"p",{},"S3 connections let you browse buckets, query objects with SQL, and use S3-compatible storage as a stream source or target.",[18,19,21],"h2",{"id":20},"supported-providers","Supported providers",[23,24,25,32,44,49,54,59,64],"ul",{},[26,27,28],"li",{},[29,30,31],"strong",{},"AWS S3",[26,33,34,37,38,43],{},[29,35,36],{},"Google Cloud Storage"," (via ",[39,40,42],"a",{"href":41},"#google-cloud-storage-s3-compatible-access","HMAC keys",")",[26,45,46],{},[29,47,48],{},"Cloudflare R2",[26,50,51],{},[29,52,53],{},"DigitalOcean Spaces",[26,55,56],{},[29,57,58],{},"Backblaze B2",[26,60,61],{},[29,62,63],{},"MinIO",[26,65,66,69],{},[29,67,68],{},"Custom"," — any S3-compatible storage API (Wasabi, Ceph, Linode Object Storage, etc.)",[18,71,73],{"id":72},"connection-settings","Connection settings",[75,76,77,93],"table",{},[78,79,80],"thead",{},[81,82,83,87,90],"tr",{},[84,85,86],"th",{},"Setting",[84,88,89],{},"Required",[84,91,92],{},"Description",[94,95,96,108,119,141,152,162,172,186],"tbody",{},[81,97,98,102,105],{},[99,100,101],"td",{},"Name",[99,103,104],{},"Yes",[99,106,107],{},"Connection label",[81,109,110,113,116],{},[99,111,112],{},"Provider preset",[99,114,115],{},"No",[99,117,118],{},"Auto-fills endpoint, region, and URL style for known providers",[81,120,121,124,127],{},[99,122,123],{},"Credential source",[99,125,126],{},"AWS only",[99,128,129,132,133,137,138],{},[29,130,131],{},"AWS Default Chain"," (environment variables → instance metadata → pod identity → ",[134,135,136],"code",{},"~\u002F.aws\u002Fcredentials",") or ",[29,139,140],{},"Static Credentials",[81,142,143,146,149],{},[99,144,145],{},"Access Key ID",[99,147,148],{},"Yes (static)",[99,150,151],{},"S3 access key. Hidden when using AWS Default Chain.",[81,153,154,157,159],{},[99,155,156],{},"Secret Access Key",[99,158,148],{},[99,160,161],{},"S3 secret key. Hidden when using AWS Default Chain.",[81,163,164,167,169],{},[99,165,166],{},"Session Token",[99,168,115],{},[99,170,171],{},"Temporary session token for STS credentials",[81,173,174,177,179],{},[99,175,176],{},"Region",[99,178,104],{},[99,180,181,182,185],{},"Bucket region (e.g. ",[134,183,184],{},"us-east-1","). Auto-detected from endpoint for some providers.",[81,187,188,191,194],{},[99,189,190],{},"Endpoint",[99,192,193],{},"Non-AWS",[99,195,196],{},"Required for non-AWS providers. Auto-filled by provider preset. Hidden for AWS with Default Chain.",[14,198,199,202],{},[29,200,201],{},"Scope (optional)"," — narrows the Data Explorer view to a specific bucket and\u002For prefix. When using this connection as a stream target, bucket and prefix are configured separately in stream settings.",[75,204,205,215],{},[78,206,207],{},[81,208,209,211,213],{},[84,210,86],{},[84,212,89],{},[84,214,92],{},[94,216,217,227],{},[81,218,219,222,224],{},[99,220,221],{},"Bucket",[99,223,115],{},[99,225,226],{},"Limits browsing to a single bucket. Required for GCS connection testing.",[81,228,229,232,234],{},[99,230,231],{},"Prefix",[99,233,115],{},[99,235,236],{},"Limits browsing to a key prefix (folder path)",[238,239,242],"alert",{"title":240,"type":241},"Quick fill","tip",[14,243,244,245],{},"Paste an S3-style URI into the connection string field to pre-populate settings. Format: ",[134,246,247],{},"s3:\u002F\u002FaccessKey:secretKey@endpoint\u002Fbucket",[18,249,251],{"id":250},"provider-presets","Provider presets",[14,253,254],{},"Each preset auto-fills the endpoint, region, and URL style:",[75,256,257,274],{},[78,258,259],{},[81,260,261,264,266,268,271],{},[84,262,263],{},"Provider",[84,265,190],{},[84,267,176],{},[84,269,270],{},"URL style",[84,272,273],{},"SSL",[94,275,276,298,318,337,357,377,397],{},[81,277,278,280,286,290,295],{},[99,279,31],{},[99,281,282],{},[283,284,285],"em",{},"(SDK default)",[99,287,288],{},[134,289,184],{},[99,291,292],{},[134,293,294],{},"auto",[99,296,297],{},"✓",[81,299,300,302,307,311,316],{},[99,301,36],{},[99,303,304],{},[134,305,306],{},"storage.googleapis.com",[99,308,309],{},[134,310,294],{},[99,312,313],{},[134,314,315],{},"path",[99,317,297],{},[81,319,320,322,327,331,335],{},[99,321,48],{},[99,323,324],{},[134,325,326],{},"\u003Caccount-id>.r2.cloudflarestorage.com",[99,328,329],{},[134,330,294],{},[99,332,333],{},[134,334,315],{},[99,336,297],{},[81,338,339,341,346,351,355],{},[99,340,53],{},[99,342,343],{},[134,344,345],{},"nyc3.digitaloceanspaces.com",[99,347,348],{},[134,349,350],{},"nyc3",[99,352,353],{},[134,354,315],{},[99,356,297],{},[81,358,359,361,366,371,375],{},[99,360,58],{},[99,362,363],{},[134,364,365],{},"s3.us-west-004.backblazeb2.com",[99,367,368],{},[134,369,370],{},"us-west-004",[99,372,373],{},[134,374,315],{},[99,376,297],{},[81,378,379,381,386,390,394],{},[99,380,63],{},[99,382,383],{},[134,384,385],{},"localhost:9000",[99,387,388],{},[134,389,184],{},[99,391,392],{},[134,393,315],{},[99,395,396],{},"✗",[81,398,399,401,406,410,414],{},[99,400,68],{},[99,402,403],{},[283,404,405],{},"(user-specified)",[99,407,408],{},[283,409,405],{},[99,411,412],{},[134,413,294],{},[99,415,297],{},[14,417,418,420,421,424],{},[29,419,48],{}," — replace ",[134,422,423],{},"\u003Caccount-id>"," with your Cloudflare account ID. Find it in the R2 dashboard. R2 does not charge egress fees.",[14,426,427,429,430,433],{},[29,428,58],{}," — use your application key ID and key. Adjust the region in the endpoint (e.g., ",[134,431,432],{},"s3.eu-central-003.backblazeb2.com",") to match your bucket's region.",[14,435,436,438,439,442],{},[29,437,63],{}," — the preset defaults to a local development instance. Set ",[134,440,441],{},"useSSL: true"," and update the endpoint for production deployments.",[14,444,445,447,448,451],{},[29,446,53],{}," — adjust the region prefix in the endpoint (e.g., ",[134,449,450],{},"sfo3.digitaloceanspaces.com",") to match your Space's region.",[18,453,455],{"id":454},"supported-file-formats","Supported file formats",[75,457,458,468],{},[78,459,460],{},[81,461,462,465],{},[84,463,464],{},"Format",[84,466,467],{},"Extensions",[94,469,470,480,500],{},[81,471,472,475],{},[99,473,474],{},"Parquet",[99,476,477],{},[134,478,479],{},".parquet",[81,481,482,485],{},[99,483,484],{},"CSV",[99,486,487,490,491,490,494,490,497],{},[134,488,489],{},".csv",", ",[134,492,493],{},".csv.gz",[134,495,496],{},".csv.zst",[134,498,499],{},".csv.bz2",[81,501,502,505],{},[99,503,504],{},"JSON \u002F JSONL",[99,506,507,490,510,490,513,490,516,490,519],{},[134,508,509],{},".json",[134,511,512],{},".jsonl",[134,514,515],{},".jsonl.gz",[134,517,518],{},".jsonl.zst",[134,520,521],{},".jsonl.bz2",[14,523,524,525,529],{},"Compressed files are detected and decompressed transparently. See ",[39,526,528],{"href":527},"\u002Fdocs\u002Fconnections\u002Fcompressed-files","Compressed File Support",".",[18,531,533],{"id":532},"typical-workflow","Typical workflow",[535,536,537,540,543,550,553],"ol",{},[26,538,539],{},"Create an S3-compatible connection.",[26,541,542],{},"Validate credentials and bucket access.",[26,544,545,546,529],{},"Open the connection in ",[39,547,549],{"href":548},"\u002Fdocs\u002Fdatabase-explorer\u002F","Data Explorer",[26,551,552],{},"Browse buckets or prefixes.",[26,554,555,556,560,561,529],{},"Query files through the ",[39,557,559],{"href":558},"\u002Fdocs\u002Fdatabase-explorer\u002Fsql-console","SQL Console"," or use them in ",[39,562,564],{"href":563},"\u002Fdocs\u002Fdatabase-explorer\u002Ffederated-queries","Federated Queries",[18,566,568],{"id":567},"reading-from-s3-source","Reading from S3 (source)",[570,571,573],"h3",{"id":572},"reading-strategies","Reading strategies",[23,575,576,585,598],{},[26,577,578,581,582],{},[29,579,580],{},"Direct file"," — point to a single S3 path: ",[134,583,584],{},"s3:\u002F\u002Fbucket\u002Fdata\u002Ffile.parquet",[26,586,587,590,591,594,595,529],{},[29,588,589],{},"Glob patterns"," — match multiple files: ",[134,592,593],{},"s3:\u002F\u002Fbucket\u002Fdata\u002F*.csv",". Works with nested directory layouts like ",[134,596,597],{},"year=2024\u002Fmonth=01\u002F*.parquet",[26,599,600,603],{},[29,601,602],{},"Manifest-driven"," — use a manifest file listing paths to read",[570,605,607],{"id":606},"manifest-files","Manifest files",[14,609,610],{},"A manifest is a JSON file that lists S3 objects belonging to a dataset. Manifests are useful for:",[23,612,613,619,625],{},[26,614,615,618],{},[29,616,617],{},"Large datasets"," (>1000 files) where S3 LIST operations become slow",[26,620,621,624],{},[29,622,623],{},"Reproducible reads"," — pin a dataset to an exact set of files",[26,626,627,630],{},[29,628,629],{},"Cross-system coordination"," — share file lists between pipelines",[632,633,635],"h4",{"id":634},"manifest-format","Manifest format",[637,638,643],"pre",{"className":639,"code":640,"language":641,"meta":642,"style":642},"language-json shiki shiki-themes github-light github-dark","{\n  \"version\": \"1.0\",\n  \"files\": [\n    \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0001.parquet\",\n    \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0002.parquet\"\n  ],\n  \"objects\": [\n    {\n      \"path\": \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0001.parquet\",\n      \"size\": 1048576,\n      \"last_modified\": \"2026-03-10T18:00:00Z\",\n      \"etag\": \"abc123\",\n      \"storage_class\": \"STANDARD\"\n    },\n    {\n      \"path\": \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0002.parquet\",\n      \"size\": 2097152,\n      \"last_modified\": \"2026-03-10T18:01:00Z\",\n      \"etag\": \"def456\",\n      \"storage_class\": \"STANDARD\"\n    }\n  ],\n  \"metadata\": {\n    \"total_size\": 3145728,\n    \"row_count\": 50000,\n    \"created_at\": \"2026-03-10T18:05:00Z\",\n    \"file_format\": \"parquet\",\n    \"stream_id\": \"stream_abc123\"\n  }\n}\n","json","",[134,644,645,654,671,680,688,694,700,708,714,727,740,753,766,777,783,788,800,812,824,836,845,851,856,865,878,891,904,917,928,934],{"__ignoreMap":642},[646,647,650],"span",{"class":648,"line":649},"line",1,[646,651,653],{"class":652},"sVt8B","{\n",[646,655,657,661,664,668],{"class":648,"line":656},2,[646,658,660],{"class":659},"sj4cs","  \"version\"",[646,662,663],{"class":652},": ",[646,665,667],{"class":666},"sZZnC","\"1.0\"",[646,669,670],{"class":652},",\n",[646,672,674,677],{"class":648,"line":673},3,[646,675,676],{"class":659},"  \"files\"",[646,678,679],{"class":652},": [\n",[646,681,683,686],{"class":648,"line":682},4,[646,684,685],{"class":666},"    \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0001.parquet\"",[646,687,670],{"class":652},[646,689,691],{"class":648,"line":690},5,[646,692,693],{"class":666},"    \"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0002.parquet\"\n",[646,695,697],{"class":648,"line":696},6,[646,698,699],{"class":652},"  ],\n",[646,701,703,706],{"class":648,"line":702},7,[646,704,705],{"class":659},"  \"objects\"",[646,707,679],{"class":652},[646,709,711],{"class":648,"line":710},8,[646,712,713],{"class":652},"    {\n",[646,715,717,720,722,725],{"class":648,"line":716},9,[646,718,719],{"class":659},"      \"path\"",[646,721,663],{"class":652},[646,723,724],{"class":666},"\"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0001.parquet\"",[646,726,670],{"class":652},[646,728,730,733,735,738],{"class":648,"line":729},10,[646,731,732],{"class":659},"      \"size\"",[646,734,663],{"class":652},[646,736,737],{"class":659},"1048576",[646,739,670],{"class":652},[646,741,743,746,748,751],{"class":648,"line":742},11,[646,744,745],{"class":659},"      \"last_modified\"",[646,747,663],{"class":652},[646,749,750],{"class":666},"\"2026-03-10T18:00:00Z\"",[646,752,670],{"class":652},[646,754,756,759,761,764],{"class":648,"line":755},12,[646,757,758],{"class":659},"      \"etag\"",[646,760,663],{"class":652},[646,762,763],{"class":666},"\"abc123\"",[646,765,670],{"class":652},[646,767,769,772,774],{"class":648,"line":768},13,[646,770,771],{"class":659},"      \"storage_class\"",[646,773,663],{"class":652},[646,775,776],{"class":666},"\"STANDARD\"\n",[646,778,780],{"class":648,"line":779},14,[646,781,782],{"class":652},"    },\n",[646,784,786],{"class":648,"line":785},15,[646,787,713],{"class":652},[646,789,791,793,795,798],{"class":648,"line":790},16,[646,792,719],{"class":659},[646,794,663],{"class":652},[646,796,797],{"class":666},"\"s3:\u002F\u002Fbucket\u002Fdata\u002Forders-0002.parquet\"",[646,799,670],{"class":652},[646,801,803,805,807,810],{"class":648,"line":802},17,[646,804,732],{"class":659},[646,806,663],{"class":652},[646,808,809],{"class":659},"2097152",[646,811,670],{"class":652},[646,813,815,817,819,822],{"class":648,"line":814},18,[646,816,745],{"class":659},[646,818,663],{"class":652},[646,820,821],{"class":666},"\"2026-03-10T18:01:00Z\"",[646,823,670],{"class":652},[646,825,827,829,831,834],{"class":648,"line":826},19,[646,828,758],{"class":659},[646,830,663],{"class":652},[646,832,833],{"class":666},"\"def456\"",[646,835,670],{"class":652},[646,837,839,841,843],{"class":648,"line":838},20,[646,840,771],{"class":659},[646,842,663],{"class":652},[646,844,776],{"class":666},[646,846,848],{"class":648,"line":847},21,[646,849,850],{"class":652},"    }\n",[646,852,854],{"class":648,"line":853},22,[646,855,699],{"class":652},[646,857,859,862],{"class":648,"line":858},23,[646,860,861],{"class":659},"  \"metadata\"",[646,863,864],{"class":652},": {\n",[646,866,868,871,873,876],{"class":648,"line":867},24,[646,869,870],{"class":659},"    \"total_size\"",[646,872,663],{"class":652},[646,874,875],{"class":659},"3145728",[646,877,670],{"class":652},[646,879,881,884,886,889],{"class":648,"line":880},25,[646,882,883],{"class":659},"    \"row_count\"",[646,885,663],{"class":652},[646,887,888],{"class":659},"50000",[646,890,670],{"class":652},[646,892,894,897,899,902],{"class":648,"line":893},26,[646,895,896],{"class":659},"    \"created_at\"",[646,898,663],{"class":652},[646,900,901],{"class":666},"\"2026-03-10T18:05:00Z\"",[646,903,670],{"class":652},[646,905,907,910,912,915],{"class":648,"line":906},27,[646,908,909],{"class":659},"    \"file_format\"",[646,911,663],{"class":652},[646,913,914],{"class":666},"\"parquet\"",[646,916,670],{"class":652},[646,918,920,923,925],{"class":648,"line":919},28,[646,921,922],{"class":659},"    \"stream_id\"",[646,924,663],{"class":652},[646,926,927],{"class":666},"\"stream_abc123\"\n",[646,929,931],{"class":648,"line":930},29,[646,932,933],{"class":652},"  }\n",[646,935,937],{"class":648,"line":936},30,[646,938,939],{"class":652},"}\n",[75,941,942,953],{},[78,943,944],{},[81,945,946,949,951],{},[84,947,948],{},"Field",[84,950,89],{},[84,952,92],{},[94,954,955,969,981,993],{},[81,956,957,962,964],{},[99,958,959],{},[134,960,961],{},"version",[99,963,104],{},[99,965,966,967,43],{},"Format version (currently ",[134,968,667],{},[81,970,971,976,978],{},[99,972,973],{},[134,974,975],{},"files",[99,977,104],{},[99,979,980],{},"Array of S3 URIs or absolute paths",[81,982,983,988,990],{},[99,984,985],{},[134,986,987],{},"objects",[99,989,115],{},[99,991,992],{},"Detailed metadata per file (size, ETag, timestamps)",[81,994,995,1000,1002],{},[99,996,997],{},[134,998,999],{},"metadata",[99,1001,115],{},[99,1003,1004],{},"Dataset-level metadata (total size, row count, creation time, stream ID)",[632,1006,1008],{"id":1007},"automatic-manifest-creation","Automatic manifest creation",[14,1010,1011,1012,1015],{},"When a ",[29,1013,1014],{},"load-mode"," stream writes to an S3 target, a manifest is automatically created at the canonical path:",[637,1017,1022],{"className":1018,"code":1020,"language":1021},[1019],"language-text","s3:\u002F\u002Fbucket\u002Fprefix\u002Fmanifest.json\n","text",[134,1023,1020],{"__ignoreMap":642},[14,1025,1026],{},"The manifest includes all uploaded files with their sizes, ETags, and storage classes.",[632,1028,1030],{"id":1029},"fast-folder-listing","Fast folder listing",[14,1032,1033,1034,1037,1038,1041],{},"When browsing S3 folders, the UI checks for a ",[134,1035,1036],{},"manifest.json"," at the canonical path before issuing an S3 LIST request. This provides instant folder loading for previously written datasets. Use the ",[29,1039,1040],{},"refresh"," button to bypass the manifest and list live bucket contents.",[632,1043,1045],{"id":1044},"using-manifests-as-a-source","Using manifests as a source",[14,1047,1048,1049,1052,1053,1056,1057,1059],{},"In stream configuration, set ",[134,1050,1051],{},"manifestPath"," instead of ",[134,1054,1055],{},"prefixes"," or ",[134,1058,987],{},":",[637,1061,1063],{"className":639,"code":1062,"language":641,"meta":642,"style":642},"{\n  \"source\": {\n    \"connections\": [\n      {\n        \"connectionId\": \"conn_s3\",\n        \"s3\": {\n          \"bucket\": \"my-data-bucket\",\n          \"manifestPath\": \"s3:\u002F\u002Fmy-data-bucket\u002Fexports\u002Fmanifest.json\"\n        }\n      }\n    ]\n  }\n}\n",[134,1064,1065,1069,1076,1083,1088,1100,1107,1119,1129,1134,1139,1144,1148],{"__ignoreMap":642},[646,1066,1067],{"class":648,"line":649},[646,1068,653],{"class":652},[646,1070,1071,1074],{"class":648,"line":656},[646,1072,1073],{"class":659},"  \"source\"",[646,1075,864],{"class":652},[646,1077,1078,1081],{"class":648,"line":673},[646,1079,1080],{"class":659},"    \"connections\"",[646,1082,679],{"class":652},[646,1084,1085],{"class":648,"line":682},[646,1086,1087],{"class":652},"      {\n",[646,1089,1090,1093,1095,1098],{"class":648,"line":690},[646,1091,1092],{"class":659},"        \"connectionId\"",[646,1094,663],{"class":652},[646,1096,1097],{"class":666},"\"conn_s3\"",[646,1099,670],{"class":652},[646,1101,1102,1105],{"class":648,"line":696},[646,1103,1104],{"class":659},"        \"s3\"",[646,1106,864],{"class":652},[646,1108,1109,1112,1114,1117],{"class":648,"line":702},[646,1110,1111],{"class":659},"          \"bucket\"",[646,1113,663],{"class":652},[646,1115,1116],{"class":666},"\"my-data-bucket\"",[646,1118,670],{"class":652},[646,1120,1121,1124,1126],{"class":648,"line":710},[646,1122,1123],{"class":659},"          \"manifestPath\"",[646,1125,663],{"class":652},[646,1127,1128],{"class":666},"\"s3:\u002F\u002Fmy-data-bucket\u002Fexports\u002Fmanifest.json\"\n",[646,1130,1131],{"class":648,"line":716},[646,1132,1133],{"class":652},"        }\n",[646,1135,1136],{"class":648,"line":729},[646,1137,1138],{"class":652},"      }\n",[646,1140,1141],{"class":648,"line":742},[646,1142,1143],{"class":652},"    ]\n",[646,1145,1146],{"class":648,"line":755},[646,1147,933],{"class":652},[646,1149,1150],{"class":648,"line":768},[646,1151,939],{"class":652},[238,1153],{"title":1154,"type":241},"`manifestPath` is mutually exclusive with `prefixes` and `objects`. Use one approach per connection.",[18,1156,1158],{"id":1157},"writing-to-s3-target","Writing to S3 (target)",[570,1160,1162],{"id":1161},"stream-target-configuration","Stream target configuration",[14,1164,1165,1166,1169],{},"When using S3 as a stream target, use ",[134,1167,1168],{},"spec.s3"," with upload settings:",[637,1171,1173],{"className":639,"code":1172,"language":641,"meta":642,"style":642},"{\n  \"target\": {\n    \"id\": \"conn_s3\",\n    \"spec\": {\n      \"s3\": {\n        \"fileFormat\": \"parquet\",\n        \"format\": {\n          \"compression\": \"zstd\"\n        },\n        \"upload\": {\n          \"bucket\": \"my-data-bucket\",\n          \"prefix\": \"exports\u002F\",\n          \"storageClass\": \"STANDARD\",\n          \"serverSideEnc\": \"AES256\"\n        }\n      }\n    }\n  }\n}\n",[134,1174,1175,1179,1186,1197,1204,1211,1222,1229,1239,1244,1251,1261,1273,1285,1295,1299,1303,1307,1311],{"__ignoreMap":642},[646,1176,1177],{"class":648,"line":649},[646,1178,653],{"class":652},[646,1180,1181,1184],{"class":648,"line":656},[646,1182,1183],{"class":659},"  \"target\"",[646,1185,864],{"class":652},[646,1187,1188,1191,1193,1195],{"class":648,"line":673},[646,1189,1190],{"class":659},"    \"id\"",[646,1192,663],{"class":652},[646,1194,1097],{"class":666},[646,1196,670],{"class":652},[646,1198,1199,1202],{"class":648,"line":682},[646,1200,1201],{"class":659},"    \"spec\"",[646,1203,864],{"class":652},[646,1205,1206,1209],{"class":648,"line":690},[646,1207,1208],{"class":659},"      \"s3\"",[646,1210,864],{"class":652},[646,1212,1213,1216,1218,1220],{"class":648,"line":696},[646,1214,1215],{"class":659},"        \"fileFormat\"",[646,1217,663],{"class":652},[646,1219,914],{"class":666},[646,1221,670],{"class":652},[646,1223,1224,1227],{"class":648,"line":702},[646,1225,1226],{"class":659},"        \"format\"",[646,1228,864],{"class":652},[646,1230,1231,1234,1236],{"class":648,"line":710},[646,1232,1233],{"class":659},"          \"compression\"",[646,1235,663],{"class":652},[646,1237,1238],{"class":666},"\"zstd\"\n",[646,1240,1241],{"class":648,"line":716},[646,1242,1243],{"class":652},"        },\n",[646,1245,1246,1249],{"class":648,"line":729},[646,1247,1248],{"class":659},"        \"upload\"",[646,1250,864],{"class":652},[646,1252,1253,1255,1257,1259],{"class":648,"line":742},[646,1254,1111],{"class":659},[646,1256,663],{"class":652},[646,1258,1116],{"class":666},[646,1260,670],{"class":652},[646,1262,1263,1266,1268,1271],{"class":648,"line":755},[646,1264,1265],{"class":659},"          \"prefix\"",[646,1267,663],{"class":652},[646,1269,1270],{"class":666},"\"exports\u002F\"",[646,1272,670],{"class":652},[646,1274,1275,1278,1280,1283],{"class":648,"line":768},[646,1276,1277],{"class":659},"          \"storageClass\"",[646,1279,663],{"class":652},[646,1281,1282],{"class":666},"\"STANDARD\"",[646,1284,670],{"class":652},[646,1286,1287,1290,1292],{"class":648,"line":779},[646,1288,1289],{"class":659},"          \"serverSideEnc\"",[646,1291,663],{"class":652},[646,1293,1294],{"class":666},"\"AES256\"\n",[646,1296,1297],{"class":648,"line":785},[646,1298,1133],{"class":652},[646,1300,1301],{"class":648,"line":790},[646,1302,1138],{"class":652},[646,1304,1305],{"class":648,"line":802},[646,1306,850],{"class":652},[646,1308,1309],{"class":648,"line":814},[646,1310,933],{"class":652},[646,1312,1313],{"class":648,"line":826},[646,1314,939],{"class":652},[75,1316,1317,1327],{},[78,1318,1319],{},[81,1320,1321,1323,1325],{},[84,1322,948],{},[84,1324,89],{},[84,1326,92],{},[94,1328,1329,1349,1376,1388,1400,1434,1453,1467],{},[81,1330,1331,1336,1338],{},[99,1332,1333],{},[134,1334,1335],{},"fileFormat",[99,1337,104],{},[99,1339,1340,1341,490,1344,490,1347],{},"Output format: ",[134,1342,1343],{},"\"csv\"",[134,1345,1346],{},"\"jsonl\"",[134,1348,914],{},[81,1350,1351,1356,1358],{},[99,1352,1353],{},[134,1354,1355],{},"format.compression",[99,1357,115],{},[99,1359,1360,1361,1364,1365,1368,1369,1372,1373,529],{},"CSV and JSONL default to ",[134,1362,1363],{},"\"uncompressed\"",". Parquet defaults to ",[134,1366,1367],{},"\"zstd\"",". Parquet also supports ",[134,1370,1371],{},"\"snappy\"","; all file targets support ",[134,1374,1375],{},"\"gzip\"",[81,1377,1378,1383,1385],{},[99,1379,1380],{},[134,1381,1382],{},"upload.bucket",[99,1384,104],{},[99,1386,1387],{},"Destination bucket",[81,1389,1390,1395,1397],{},[99,1391,1392],{},[134,1393,1394],{},"upload.prefix",[99,1396,115],{},[99,1398,1399],{},"Key prefix for file organization",[81,1401,1402,1407,1409],{},[99,1403,1404],{},[134,1405,1406],{},"upload.storageClass",[99,1408,115],{},[99,1410,1411,1412,1415,1416,490,1419,490,1422,490,1425,490,1428,490,1431,529],{},"Default: ",[134,1413,1414],{},"STANDARD",". Options: ",[134,1417,1418],{},"INTELLIGENT_TIERING",[134,1420,1421],{},"STANDARD_IA",[134,1423,1424],{},"ONEZONE_IA",[134,1426,1427],{},"GLACIER_IR",[134,1429,1430],{},"GLACIER",[134,1432,1433],{},"DEEP_ARCHIVE",[81,1435,1436,1441,1443],{},[99,1437,1438],{},[134,1439,1440],{},"upload.serverSideEnc",[99,1442,115],{},[99,1444,1445,1448,1449,1452],{},[134,1446,1447],{},"AES256"," (SSE-S3) or ",[134,1450,1451],{},"aws:kms"," (SSE-KMS)",[81,1454,1455,1460,1462],{},[99,1456,1457],{},[134,1458,1459],{},"upload.kmsKeyId",[99,1461,115],{},[99,1463,1464,1465],{},"KMS key ID when using ",[134,1466,1451],{},[81,1468,1469,1474,1476],{},[99,1470,1471],{},[134,1472,1473],{},"upload.keepLocalFiles",[99,1475,115],{},[99,1477,1478,1479,1482],{},"Retain staging files on disk after upload (default: ",[134,1480,1481],{},"false","). Useful for local backup or debugging.",[14,1484,1485,1486,1489],{},"File format is specified in the ",[29,1487,1488],{},"target spec",", not the connection. One S3 connection can serve multiple streams with different formats.",[570,1491,1493],{"id":1492},"output-organization","Output organization",[14,1495,1496,1497],{},"Files are organized in Hive-style partitioning: ",[134,1498,1499],{},"\u003Cprefix>\u002F\u003Ctable>\u002Fpart-\u003Cinstance>-\u003Cseq>.\u003Cext>",[14,1501,1502],{},"Features:",[23,1504,1505,1508,1511,1514],{},[26,1506,1507],{},"Parallel streaming from multiple writer instances",[26,1509,1510],{},"Automatic file rotation at 200 MB",[26,1512,1513],{},"Multipart upload for files over 64 MB",[26,1515,1516],{},"Compatible with Athena, Snowflake, Spark, and Hive",[18,1518,1520],{"id":1519},"google-cloud-storage-s3-compatible-access","Google Cloud Storage (S3-compatible access)",[14,1522,1523],{},"GCS supports S3-compatible access via HMAC keys. This lets you use GCS buckets with the same S3 connection type.",[570,1525,1527],{"id":1526},"setup","Setup",[535,1529,1530,1539,1549,1556],{},[26,1531,1532,1533],{},"Go to ",[39,1534,1538],{"href":1535,"rel":1536},"https:\u002F\u002Fconsole.cloud.google.com\u002Fstorage\u002Fsettings;tab=interoperability",[1537],"nofollow","Cloud Storage → Settings → Interoperability",[26,1540,1541,1542,1545,1546],{},"Under ",[29,1543,1544],{},"Access keys for service accounts",", click ",[29,1547,1548],{},"Create a key for a service account",[26,1550,1551,1552,1555],{},"Select your service account (or create one with ",[29,1553,1554],{},"Storage Object Admin"," role)",[26,1557,1558,1559,1562,1563,1566,1567,1570],{},"Copy the ",[29,1560,1561],{},"Access Key"," (",[134,1564,1565],{},"GOOG1E...",") and ",[29,1568,1569],{},"Secret"," immediately — the secret is shown only once",[570,1572,73],{"id":1573},"connection-settings-1",[14,1575,1576,1577,1579,1580,1582],{},"Select the ",[29,1578,36],{}," preset, then enter your HMAC Access Key (",[134,1581,1565],{},") and Secret as static credentials. The endpoint and region are auto-filled.",[238,1584],{"title":1585,"type":1586},"Google Cloud Storage **requires a bucket name** for connection testing. Unlike AWS S3, GCS doesn't support the `ListBuckets` operation via the S3 interoperability API. Enter your bucket name in the Scope section.","warning",[570,1588,1590],{"id":1589},"permissions","Permissions",[14,1592,1593],{},"Minimum required:",[23,1595,1596],{},[26,1597,1598,490,1601,490,1604,490,1607,490,1610],{},[134,1599,1600],{},"storage.objects.create",[134,1602,1603],{},"storage.objects.delete",[134,1605,1606],{},"storage.objects.get",[134,1608,1609],{},"storage.objects.list",[134,1611,1612],{},"storage.buckets.get",[14,1614,1615,1616,1562,1618,43],{},"Recommended role: ",[29,1617,1554],{},[134,1619,1620],{},"roles\u002Fstorage.objectAdmin",[570,1622,1624],{"id":1623},"key-rotation","Key rotation",[535,1626,1627,1630,1633,1636],{},[26,1628,1629],{},"Create a new HMAC key",[26,1631,1632],{},"Update your DBConvert Streams connection with the new credentials",[26,1634,1635],{},"Verify the connection works",[26,1637,1638],{},"Delete the old key",[14,1640,1641,1642,1647,1648,529],{},"For more details, see ",[39,1643,1646],{"href":1644,"rel":1645},"https:\u002F\u002Fcloud.google.com\u002Fstorage\u002Fdocs\u002Finteroperability",[1537],"Google Cloud Storage Interoperability"," and ",[39,1649,1652],{"href":1650,"rel":1651},"https:\u002F\u002Fcloud.google.com\u002Fstorage\u002Fdocs\u002Fauthentication\u002Fhmackeys",[1537],"HMAC Keys for Service Accounts",[18,1654,1656],{"id":1655},"constraints","Constraints",[23,1658,1659,1662],{},[26,1660,1661],{},"S3 source workflows are query-oriented (Load mode, Explorer, SQL). They are not CDC sources.",[26,1663,1664],{},"Provider-specific extensions outside the S3-compatible API model are not guaranteed.",[18,1666,1668],{"id":1667},"related-docs","Related docs",[23,1670,1671,1677,1683,1687,1693],{},[26,1672,1673],{},[39,1674,1676],{"href":1675},"\u002Fdocs\u002Fconnections\u002Fconnection-management","Connection Management",[26,1678,1679],{},[39,1680,1682],{"href":1681},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Stream Configuration Reference",[26,1684,1685],{},[39,1686,528],{"href":527},[26,1688,1689],{},[39,1690,1692],{"href":1691},"\u002Fdocs\u002Fconnections\u002Flocal-files","Local Files",[26,1694,1695],{},[39,1696,564],{"href":563},[1698,1699,1700],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":642,"searchDepth":656,"depth":656,"links":1702},[1703,1704,1705,1706,1707,1708,1712,1716,1722,1723],{"id":20,"depth":656,"text":21},{"id":72,"depth":656,"text":73},{"id":250,"depth":656,"text":251},{"id":454,"depth":656,"text":455},{"id":532,"depth":656,"text":533},{"id":567,"depth":656,"text":568,"children":1709},[1710,1711],{"id":572,"depth":673,"text":573},{"id":606,"depth":673,"text":607},{"id":1157,"depth":656,"text":1158,"children":1713},[1714,1715],{"id":1161,"depth":673,"text":1162},{"id":1492,"depth":673,"text":1493},{"id":1519,"depth":656,"text":1520,"children":1717},[1718,1719,1720,1721],{"id":1526,"depth":673,"text":1527},{"id":1573,"depth":673,"text":73},{"id":1589,"depth":673,"text":1590},{"id":1623,"depth":673,"text":1624},{"id":1655,"depth":656,"text":1656},{"id":1667,"depth":656,"text":1668},"Work with S3-compatible object storage for browsing, SQL, file inspection, and stream targets in DBConvert Streams.","md",{},false,"\u002Fdocs\u002Fconnections\u002Fs3-compatible-storage",null,{"title":5,"description":1724},"docs\u002Fconnections\u002Fs3-compatible-storage","27TWGgzUr89FP32NhtPJAo_KM2ekWyJGOjkpmQu9Cbo",1783896284384]