[{"data":1,"prerenderedAt":3559},["ShallowReactive",2],{"navigation":3,"\u002Fobserve\u002Fguides\u002Fbuilding-an-etl":750,"\u002Fobserve\u002Fguides\u002Fbuilding-an-etl-surround":3554},[4,80,134,364,428,530,567,646],{"title":5,"path":6,"stem":7,"children":8,"page":42},"Get Started","\u002Fget-started","01.get-started",[9,13,17,43,47],{"title":10,"path":11,"stem":12},"Welcome to UniteLabs","\u002Fget-started\u002Fwelcome","01.get-started\u002F00.welcome",{"title":14,"path":15,"stem":16},"How it works","\u002Fget-started\u002Fhow-it-works","01.get-started\u002F01.how-it-works",{"title":18,"path":19,"stem":20,"children":21,"page":42},"Use Cases","\u002Fget-started\u002Fuse-cases","01.get-started\u002F02.use-cases",[22,26,30,34,38],{"title":23,"path":24,"stem":25},"Instrument Control","\u002Fget-started\u002Fuse-cases\u002Flow-level-instrument-control","01.get-started\u002F02.use-cases\u002F10.low-level-instrument-control",{"title":27,"path":28,"stem":29},"Multi-device Control","\u002Fget-started\u002Fuse-cases\u002Fmulti-device-control","01.get-started\u002F02.use-cases\u002F20.multi-device-control",{"title":31,"path":32,"stem":33},"Workflow Orchestration","\u002Fget-started\u002Fuse-cases\u002Fworkflow-orchestration","01.get-started\u002F02.use-cases\u002F30.workflow-orchestration",{"title":35,"path":36,"stem":37},"Custom Apps and LIMS Integration","\u002Fget-started\u002Fuse-cases\u002Fcustom-apps-lims","01.get-started\u002F02.use-cases\u002F40.custom-apps-lims",{"title":39,"path":40,"stem":41},"Scale and Fleet Management","\u002Fget-started\u002Fuse-cases\u002Fscale-and-fleet","01.get-started\u002F02.use-cases\u002F50.scale-and-fleet",false,{"title":44,"path":45,"stem":46},"Lab as Code","\u002Fget-started\u002Fwhy-code","01.get-started\u002F03.why-code",{"title":48,"path":49,"stem":50,"children":51,"page":42},"Setup","\u002Fget-started\u002Fsetup","01.get-started\u002F20.setup",[52,56,60,64,68,72,76],{"title":53,"path":54,"stem":55},"GroundControl","\u002Fget-started\u002Fsetup\u002Fgroundcontrol","01.get-started\u002F20.setup\u002F10.groundcontrol",{"title":57,"path":58,"stem":59},"Network requirements","\u002Fget-started\u002Fsetup\u002Fnetwork-requirements","01.get-started\u002F20.setup\u002F20.network-requirements",{"title":61,"path":62,"stem":63},"Multi-device Networking","\u002Fget-started\u002Fsetup\u002Fmultiple-devices","01.get-started\u002F20.setup\u002F30.multiple-devices",{"title":65,"path":66,"stem":67},"Connect to Platform","\u002Fget-started\u002Fsetup\u002Fconnect-to-platform","01.get-started\u002F20.setup\u002F40.connect-to-platform",{"title":69,"path":70,"stem":71},"Headless install","\u002Fget-started\u002Fsetup\u002Fheadless-install","01.get-started\u002F20.setup\u002F50.headless-install",{"title":73,"path":74,"stem":75},"Raspberry Pi","\u002Fget-started\u002Fsetup\u002Fraspberry-pi","01.get-started\u002F20.setup\u002F55.raspberry-pi",{"title":77,"path":78,"stem":79},"Installation","\u002Fget-started\u002Fsetup\u002Fsdk-installation","01.get-started\u002F20.setup\u002F60.sdk-installation",{"title":81,"path":82,"stem":83,"children":84,"page":42},"Integrate","\u002Fintegrate","02.integrate",[85,89,93,118],{"title":86,"path":87,"stem":88},"What is a connector?","\u002Fintegrate\u002Fwhat-is-a-connector","02.integrate\u002F00.what-is-a-connector",{"title":90,"path":91,"stem":92},"Calling a Connector","\u002Fintegrate\u002Fcontrol-with-code","02.integrate\u002F05.control-with-code",{"title":94,"path":95,"stem":96,"children":97,"page":42},"Concepts","\u002Fintegrate\u002Fconcepts","02.integrate\u002F10.concepts",[98,102,106,110,114],{"title":99,"path":100,"stem":101},"Connector","\u002Fintegrate\u002Fconcepts\u002Fconnector","02.integrate\u002F10.concepts\u002F10.connector",{"title":103,"path":104,"stem":105},"Module","\u002Fintegrate\u002Fconcepts\u002Fmodule","02.integrate\u002F10.concepts\u002F20.module",{"title":107,"path":108,"stem":109},"Action","\u002Fintegrate\u002Fconcepts\u002Faction","02.integrate\u002F10.concepts\u002F30.action",{"title":111,"path":112,"stem":113},"Subscription","\u002Fintegrate\u002Fconcepts\u002Fsubscription","02.integrate\u002F10.concepts\u002F40.subscription",{"title":115,"path":116,"stem":117},"Device","\u002Fintegrate\u002Fconcepts\u002Fdevice","02.integrate\u002F10.concepts\u002F50.device",{"title":119,"path":120,"stem":121,"children":122,"page":42},"Use a connector","\u002Fintegrate\u002Fuse-a-connector","02.integrate\u002F30.use-a-connector",[123,126,130],{"title":53,"path":124,"stem":125},"\u002Fintegrate\u002Fuse-a-connector\u002Fgroundcontrol","02.integrate\u002F30.use-a-connector\u002F10.groundcontrol",{"title":127,"path":128,"stem":129},"Platform","\u002Fintegrate\u002Fuse-a-connector\u002Fplatform","02.integrate\u002F30.use-a-connector\u002F20.platform",{"title":131,"path":132,"stem":133},"UniteLabs SDK & REST API","\u002Fintegrate\u002Fuse-a-connector\u002Fpython","02.integrate\u002F30.use-a-connector\u002F30.python",{"title":135,"path":136,"stem":137,"children":138,"page":42},"Operate","\u002Foperate","03.operate",[139,142,146,186,259],{"title":135,"path":140,"stem":141},"\u002Foperate\u002Foverview","03.operate\u002F00.overview",{"title":143,"path":144,"stem":145},"Your First Protocol","\u002Foperate\u002Fyour-first-protocol","03.operate\u002F05.your-first-protocol",{"title":94,"path":147,"stem":148,"children":149,"page":42},"\u002Foperate\u002Fconcepts","03.operate\u002F10.concepts",[150,154,158,162,166,170,174,178,182],{"title":151,"path":152,"stem":153},"Liquid Handling","\u002Foperate\u002Fconcepts\u002Fliquid-handling","03.operate\u002F10.concepts\u002F10.liquid-handling",{"title":155,"path":156,"stem":157},"Modules","\u002Foperate\u002Fconcepts\u002Fmodules","03.operate\u002F10.concepts\u002F20.modules",{"title":159,"path":160,"stem":161},"Deck","\u002Foperate\u002Fconcepts\u002Fdeck","03.operate\u002F10.concepts\u002F30.deck",{"title":163,"path":164,"stem":165},"Labware","\u002Foperate\u002Fconcepts\u002Flabware","03.operate\u002F10.concepts\u002F40.labware",{"title":167,"path":168,"stem":169},"Liquids","\u002Foperate\u002Fconcepts\u002Fliquids","03.operate\u002F10.concepts\u002F50.liquids",{"title":171,"path":172,"stem":173},"Liquid Classes","\u002Foperate\u002Fconcepts\u002Fliquid-classes","03.operate\u002F10.concepts\u002F60.liquid-classes",{"title":175,"path":176,"stem":177},"Tips and Tip Tracking","\u002Foperate\u002Fconcepts\u002Ftips","03.operate\u002F10.concepts\u002F70.tips",{"title":179,"path":180,"stem":181},"Simulation","\u002Foperate\u002Fconcepts\u002Fsimulation","03.operate\u002F10.concepts\u002F80.simulation",{"title":183,"path":184,"stem":185},"Error Handling","\u002Foperate\u002Fconcepts\u002Ferror-handling","03.operate\u002F10.concepts\u002F90.error-handling",{"title":187,"path":188,"stem":189,"children":190,"page":42},"Guides","\u002Foperate\u002Fguides","03.operate\u002F20.guides",[191,220,232,256],{"title":163,"path":192,"stem":193,"children":194},"\u002Foperate\u002Fguides\u002Flabware","03.operate\u002F20.guides\u002F10.labware\u002Findex",[195,196,200,204,208,212,216],{"title":163,"path":192,"stem":193},{"title":197,"path":198,"stem":199},"Standard Labware","\u002Foperate\u002Fguides\u002Flabware\u002Fstandard-labware","03.operate\u002F20.guides\u002F10.labware\u002F05.standard-labware",{"title":201,"path":202,"stem":203},"Plates","\u002Foperate\u002Fguides\u002Flabware\u002Fplates","03.operate\u002F20.guides\u002F10.labware\u002F10.plates",{"title":205,"path":206,"stem":207},"Tips and Tip Racks","\u002Foperate\u002Fguides\u002Flabware\u002Ftips-and-racks","03.operate\u002F20.guides\u002F10.labware\u002F20.tips-and-racks",{"title":209,"path":210,"stem":211},"Tubes and Tube Racks","\u002Foperate\u002Fguides\u002Flabware\u002Ftubes-and-tube-racks","03.operate\u002F20.guides\u002F10.labware\u002F30.tubes-and-tube-racks",{"title":213,"path":214,"stem":215},"Troughs","\u002Foperate\u002Fguides\u002Flabware\u002Ftroughs","03.operate\u002F20.guides\u002F10.labware\u002F35.troughs",{"title":217,"path":218,"stem":219},"Carriers and Adapters","\u002Foperate\u002Fguides\u002Flabware\u002Fcarriers-and-adapters","03.operate\u002F20.guides\u002F10.labware\u002F40.carriers-and-adapters",{"title":159,"path":221,"stem":222,"children":223,"page":42},"\u002Foperate\u002Fguides\u002Fdeck","03.operate\u002F20.guides\u002F20.deck",[224,228],{"title":225,"path":226,"stem":227},"Building a Deck","\u002Foperate\u002Fguides\u002Fdeck\u002Fbuilding-a-deck","03.operate\u002F20.guides\u002F20.deck\u002F10.building-a-deck",{"title":229,"path":230,"stem":231},"Save\u002FLoad a Deck","\u002Foperate\u002Fguides\u002Fdeck\u002Fsave-load-deck","03.operate\u002F20.guides\u002F20.deck\u002F20.save-load-deck",{"title":233,"path":234,"stem":235,"children":236,"page":42},"Pipetting","\u002Foperate\u002Fguides\u002Fpipetting","03.operate\u002F20.guides\u002F30.pipetting",[237,241,245,249,252],{"title":238,"path":239,"stem":240},"Basic Pipetting","\u002Foperate\u002Fguides\u002Fpipetting\u002Fbasic-pipetting","03.operate\u002F20.guides\u002F30.pipetting\u002F10.basic-pipetting",{"title":242,"path":243,"stem":244},"Advanced Pipetting","\u002Foperate\u002Fguides\u002Fpipetting\u002Fadvanced-pipetting","03.operate\u002F20.guides\u002F30.pipetting\u002F20.advanced-pipetting",{"title":246,"path":247,"stem":248},"Tip Handling","\u002Foperate\u002Fguides\u002Fpipetting\u002Ftip-handling","03.operate\u002F20.guides\u002F30.pipetting\u002F30.tip-handling",{"title":171,"path":250,"stem":251},"\u002Foperate\u002Fguides\u002Fpipetting\u002Fliquid-classes","03.operate\u002F20.guides\u002F30.pipetting\u002F40.liquid-classes",{"title":253,"path":254,"stem":255},"Labware Transport","\u002Foperate\u002Fguides\u002Fpipetting\u002Flabware-transport","03.operate\u002F20.guides\u002F30.pipetting\u002F50.labware-transport",{"title":179,"path":257,"stem":258},"\u002Foperate\u002Fguides\u002Fsimulation","03.operate\u002F20.guides\u002F40.simulation",{"title":260,"path":261,"stem":262,"children":263,"page":42},"Liquid Handler Guides","\u002Foperate\u002Fdevices","03.operate\u002F30.devices",[264,305,330,347],{"title":265,"path":266,"stem":267,"children":268,"page":42},"Hamilton STAR","\u002Foperate\u002Fdevices\u002Fhamilton-star","03.operate\u002F30.devices\u002F10.hamilton-star",[269,273,277,281,285,289,293,297,301],{"title":270,"path":271,"stem":272},"Positioning & Movement","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fpositioning","03.operate\u002F30.devices\u002F10.hamilton-star\u002F10.positioning",{"title":274,"path":275,"stem":276},"Gripper Module","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fcore-gripper","03.operate\u002F30.devices\u002F10.hamilton-star\u002F20.core-gripper",{"title":278,"path":279,"stem":280},"iSWAP Module","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fiswap","03.operate\u002F30.devices\u002F10.hamilton-star\u002F30.iswap",{"title":282,"path":283,"stem":284},"Using TADM","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Ftadm","03.operate\u002F30.devices\u002F10.hamilton-star\u002F40.tadm",{"title":286,"path":287,"stem":288},"Autoload Module","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fautoload","03.operate\u002F30.devices\u002F10.hamilton-star\u002F50.autoload",{"title":290,"path":291,"stem":292},"Waste Block","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fwaste-block","03.operate\u002F30.devices\u002F10.hamilton-star\u002F60.waste-block",{"title":294,"path":295,"stem":296},"Complex Mixing","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fcomplex-mixing","03.operate\u002F30.devices\u002F10.hamilton-star\u002F66.complex-mixing",{"title":298,"path":299,"stem":300},"Teaching Gripper Paddle Positions","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fgripper-positions","03.operate\u002F30.devices\u002F10.hamilton-star\u002F70.gripper-positions",{"title":302,"path":303,"stem":304},"Training a Custom Deck Position With the iSWAP","\u002Foperate\u002Fdevices\u002Fhamilton-star\u002Fcustom-deck-position","03.operate\u002F30.devices\u002F10.hamilton-star\u002F75.custom-deck-position",{"title":306,"path":307,"stem":308,"children":309,"page":42},"Hamilton Vantage","\u002Foperate\u002Fdevices\u002Fhamilton-vantage","03.operate\u002F30.devices\u002F20.hamilton-vantage",[310,314,318,322,326],{"title":311,"path":312,"stem":313},"Waste Configuration","\u002Foperate\u002Fdevices\u002Fhamilton-vantage\u002Fwaste-configuration","03.operate\u002F30.devices\u002F20.hamilton-vantage\u002F05.waste-configuration",{"title":315,"path":316,"stem":317},"Covers and Lights","\u002Foperate\u002Fdevices\u002Fhamilton-vantage\u002Fcovers-and-lights","03.operate\u002F30.devices\u002F20.hamilton-vantage\u002F10.covers-and-lights",{"title":319,"path":320,"stem":321},"IDL Module","\u002Foperate\u002Fdevices\u002Fhamilton-vantage\u002Fidl","03.operate\u002F30.devices\u002F20.hamilton-vantage\u002F20.idl",{"title":323,"path":324,"stem":325},"IPG Module","\u002Foperate\u002Fdevices\u002Fhamilton-vantage\u002Fipg","03.operate\u002F30.devices\u002F20.hamilton-vantage\u002F30.ipg",{"title":327,"path":328,"stem":329},"XL Pipettes","\u002Foperate\u002Fdevices\u002Fhamilton-vantage\u002Fxl-channels","03.operate\u002F30.devices\u002F20.hamilton-vantage\u002F40.xl-channels",{"title":331,"path":332,"stem":333,"children":334,"page":42},"Agilent Bravo","\u002Foperate\u002Fdevices\u002Fagilent-bravo","03.operate\u002F30.devices\u002F30.agilent-bravo",[335,338,341,344],{"title":246,"path":336,"stem":337},"\u002Foperate\u002Fdevices\u002Fagilent-bravo\u002Ftip-handling","03.operate\u002F30.devices\u002F30.agilent-bravo\u002F10.tip-handling",{"title":238,"path":339,"stem":340},"\u002Foperate\u002Fdevices\u002Fagilent-bravo\u002Fbasic-pipetting","03.operate\u002F30.devices\u002F30.agilent-bravo\u002F20.basic-pipetting",{"title":171,"path":342,"stem":343},"\u002Foperate\u002Fdevices\u002Fagilent-bravo\u002Fliquid-classes","03.operate\u002F30.devices\u002F30.agilent-bravo\u002F25.liquid-classes",{"title":274,"path":345,"stem":346},"\u002Foperate\u002Fdevices\u002Fagilent-bravo\u002Fusing-the-gripper","03.operate\u002F30.devices\u002F30.agilent-bravo\u002F30.using-the-gripper",{"title":348,"path":349,"stem":350,"children":351,"page":42},"Tecan FluentControl","\u002Foperate\u002Fdevices\u002Ftecan-fluentcontrol","03.operate\u002F30.devices\u002F40.tecan-fluentcontrol",[352,356,360],{"title":353,"path":354,"stem":355},"Connect & Run a Method","\u002Foperate\u002Fdevices\u002Ftecan-fluentcontrol\u002Fconnect-and-run","03.operate\u002F30.devices\u002F40.tecan-fluentcontrol\u002F10.connect-and-run",{"title":357,"path":358,"stem":359},"Sending XML Commands","\u002Foperate\u002Fdevices\u002Ftecan-fluentcontrol\u002Fsending-xml-commands","03.operate\u002F30.devices\u002F40.tecan-fluentcontrol\u002F20.sending-xml-commands",{"title":361,"path":362,"stem":363},"Full Workflow Example","\u002Foperate\u002Fdevices\u002Ftecan-fluentcontrol\u002Ffull-workflow-example","03.operate\u002F30.devices\u002F40.tecan-fluentcontrol\u002F30.full-workflow-example",{"title":365,"path":366,"stem":367,"children":368,"page":42},"Device Guides","\u002Fdevice-guides","06.device-guides",[369,372,385,402,415],{"title":365,"path":370,"stem":371},"\u002Fdevice-guides\u002Foverview","06.device-guides\u002F00.overview",{"title":373,"path":374,"stem":375,"children":376,"page":42},"Incubation","\u002Fdevice-guides\u002Fincubation","06.device-guides\u002F20.incubation",[377,381],{"title":378,"path":379,"stem":380},"Inheco ODTC","\u002Fdevice-guides\u002Fincubation\u002Finheco-odtc","06.device-guides\u002F20.incubation\u002F10.inheco-odtc",{"title":382,"path":383,"stem":384},"Inheco SCILA","\u002Fdevice-guides\u002Fincubation\u002Finheco-scila","06.device-guides\u002F20.incubation\u002F20.inheco-scila",{"title":386,"path":387,"stem":388,"children":389,"page":42},"Heater-Shakers","\u002Fdevice-guides\u002Fheater-shakers","06.device-guides\u002F30.heater-shakers",[390,394,398],{"title":391,"path":392,"stem":393},"Inheco Thermoshake","\u002Fdevice-guides\u002Fheater-shakers\u002Finheco-thermoshake","06.device-guides\u002F30.heater-shakers\u002F10.inheco-thermoshake",{"title":395,"path":396,"stem":397},"QInstruments Bioshake","\u002Fdevice-guides\u002Fheater-shakers\u002Fqinstruments-bioshake","06.device-guides\u002F30.heater-shakers\u002F20.qinstruments-bioshake",{"title":399,"path":400,"stem":401},"QInstruments ColdPlate","\u002Fdevice-guides\u002Fheater-shakers\u002Fqinstruments-coldplate","06.device-guides\u002F30.heater-shakers\u002F30.qinstruments-coldplate",{"title":403,"path":404,"stem":405,"children":406,"page":42},"Readers","\u002Fdevice-guides\u002Freaders","06.device-guides\u002F40.readers",[407,411],{"title":408,"path":409,"stem":410},"Biotek Gen5","\u002Fdevice-guides\u002Freaders\u002Fbiotek-gen5","06.device-guides\u002F40.readers\u002F10.biotek-gen5",{"title":412,"path":413,"stem":414},"Biotek Synergy H1","\u002Fdevice-guides\u002Freaders\u002Fbiotek-synergy-h1","06.device-guides\u002F40.readers\u002F20.biotek-synergy-H1",{"title":416,"path":417,"stem":418,"children":419,"page":42},"Tilt & Turn","\u002Fdevice-guides\u002Ftilt-and-turn","06.device-guides\u002F50.tilt-and-turn",[420,424],{"title":421,"path":422,"stem":423},"TurnStation","\u002Fdevice-guides\u002Ftilt-and-turn\u002Fqinstruments-turnstation","06.device-guides\u002F50.tilt-and-turn\u002F10.qinstruments-turnstation",{"title":425,"path":426,"stem":427},"TiltStation","\u002Fdevice-guides\u002Ftilt-and-turn\u002Fqinstruments-tiltstation","06.device-guides\u002F50.tilt-and-turn\u002F20.qinstruments-tiltstation",{"title":429,"path":430,"stem":431,"children":432,"page":42},"Automate","\u002Fautomate","07.automate",[433,437,441,445,494],{"title":434,"path":435,"stem":436},"What is a workflow?","\u002Fautomate\u002Fwhat-is-a-workflow","07.automate\u002F00.what-is-a-workflow",{"title":438,"path":439,"stem":440},"Workflow template","\u002Fautomate\u002Fworkflow-template","07.automate\u002F05.workflow-template",{"title":442,"path":443,"stem":444},"Your First Workflow","\u002Fautomate\u002Fyour-first-workflow","07.automate\u002F08.your-first-workflow",{"title":94,"pages":446,"path":456,"stem":457,"children":458,"page":42},[447,448,449,450,451,452,453,454,455],"workflow","phase","step","runs","input","artifacts","logs","human-in-the-loop","error-handling","\u002Fautomate\u002Fconcepts","07.automate\u002F10.concepts",[459,463,467,471,475,479,483,487,491],{"title":460,"path":461,"stem":462},"Workflow","\u002Fautomate\u002Fconcepts\u002Fworkflow","07.automate\u002F10.concepts\u002F10.workflow",{"title":464,"path":465,"stem":466},"Phase","\u002Fautomate\u002Fconcepts\u002Fphase","07.automate\u002F10.concepts\u002F20.phase",{"title":468,"path":469,"stem":470},"Step","\u002Fautomate\u002Fconcepts\u002Fstep","07.automate\u002F10.concepts\u002F30.step",{"title":472,"path":473,"stem":474},"Runs","\u002Fautomate\u002Fconcepts\u002Fruns","07.automate\u002F10.concepts\u002F40.runs",{"title":476,"path":477,"stem":478},"Input","\u002Fautomate\u002Fconcepts\u002Finput","07.automate\u002F10.concepts\u002F50.input",{"title":480,"path":481,"stem":482},"Artifacts","\u002Fautomate\u002Fconcepts\u002Fartifacts","07.automate\u002F10.concepts\u002F60.artifacts",{"title":484,"path":485,"stem":486},"Logs","\u002Fautomate\u002Fconcepts\u002Flogs","07.automate\u002F10.concepts\u002F70.logs",{"title":488,"path":489,"stem":490},"Human in the Loop","\u002Fautomate\u002Fconcepts\u002Fhuman-in-the-loop","07.automate\u002F10.concepts\u002F80.human-in-the-loop",{"title":183,"path":492,"stem":493},"\u002Fautomate\u002Fconcepts\u002Ferror-handling","07.automate\u002F10.concepts\u002F90.error-handling",{"title":187,"path":495,"stem":496,"children":497,"page":42},"\u002Fautomate\u002Fguides","07.automate\u002F25.guides",[498,502,506,510,514,518,522,526],{"title":499,"path":500,"stem":501},"Deploy a workflow","\u002Fautomate\u002Fguides\u002Fdeploy-a-workflow","07.automate\u002F25.guides\u002F30.deploy-a-workflow",{"title":503,"path":504,"stem":505},"Run a workflow","\u002Fautomate\u002Fguides\u002Frun-a-workflow","07.automate\u002F25.guides\u002F35.run-a-workflow",{"title":507,"path":508,"stem":509},"CI\u002FCD for workflows","\u002Fautomate\u002Fguides\u002Fcicd-for-workflows","07.automate\u002F25.guides\u002F40.cicd-for-workflows",{"title":511,"path":512,"stem":513},"HITL basics","\u002Fautomate\u002Fguides\u002Fbasic-hitl","07.automate\u002F25.guides\u002F50.basic-hitl",{"title":515,"path":516,"stem":517},"Typed operator inputs","\u002Fautomate\u002Fguides\u002Ftyped-operator-inputs","07.automate\u002F25.guides\u002F55.typed-operator-inputs",{"title":519,"path":520,"stem":521},"Error recovery","\u002Fautomate\u002Fguides\u002Fbasic-error-handling","07.automate\u002F25.guides\u002F60.basic-error-handling",{"title":523,"path":524,"stem":525},"Advanced error handling","\u002Fautomate\u002Fguides\u002Fadvanced-error-handling","07.automate\u002F25.guides\u002F70.advanced-error-handling",{"title":527,"path":528,"stem":529},"Workflows REST API","\u002Fautomate\u002Fguides\u002Fworkflows-api","07.automate\u002F25.guides\u002F80.workflows-api",{"title":531,"path":532,"stem":533,"children":534,"page":42},"Observe","\u002Fobserve","08.observe",[535,539,555],{"title":536,"path":537,"stem":538},"Overview","\u002Fobserve\u002Foverview","08.observe\u002F10.overview",{"title":94,"path":540,"stem":541,"children":542,"page":42},"\u002Fobserve\u002Fconcepts","08.observe\u002F20.concepts",[543,547,551],{"title":544,"path":545,"stem":546},"Object Storage","\u002Fobserve\u002Fconcepts\u002Fstorage","08.observe\u002F20.concepts\u002F10.storage",{"title":548,"path":549,"stem":550},"Secrets","\u002Fobserve\u002Fconcepts\u002Fsecrets","08.observe\u002F20.concepts\u002F20.secrets",{"title":552,"path":553,"stem":554},"Data sources","\u002Fobserve\u002Fconcepts\u002Fdata-sources","08.observe\u002F20.concepts\u002F30.data-sources",{"title":187,"path":556,"stem":557,"children":558,"page":42},"\u002Fobserve\u002Fguides","08.observe\u002F30.guides",[559,563],{"title":560,"path":561,"stem":562},"File System Connector","\u002Fobserve\u002Fguides\u002Ffilesystem-connector","08.observe\u002F30.guides\u002F10.filesystem-connector",{"title":564,"path":565,"stem":566},"Building an ETL","\u002Fobserve\u002Fguides\u002Fbuilding-an-etl","08.observe\u002F30.guides\u002F20.building-an-etl",{"title":568,"path":569,"stem":570,"children":571,"page":42},"Technical Reference","\u002Ftechnical-reference","10.technical-reference",[572,576,580,592,603,614,624,635],{"title":573,"path":574,"stem":575},"Introduction","\u002Ftechnical-reference\u002Fplatform-blueprint","10.technical-reference\u002F00.platform-blueprint",{"title":577,"path":578,"stem":579},"REST API","\u002Ftechnical-reference\u002Frest-api","10.technical-reference\u002F01.rest-api",{"title":53,"path":581,"stem":582,"children":583,"page":42},"\u002Ftechnical-reference\u002Fgroundcontrol","10.technical-reference\u002F10.groundcontrol",[584,588],{"title":585,"path":586,"stem":587},"Install & Setup","\u002Ftechnical-reference\u002Fgroundcontrol\u002Finstall","10.technical-reference\u002F10.groundcontrol\u002F10.install",{"title":589,"path":590,"stem":591},"Changelog","\u002Ftechnical-reference\u002Fgroundcontrol\u002Fchangelog","10.technical-reference\u002F10.groundcontrol\u002F20.changelog",{"title":593,"path":594,"stem":595,"children":596,"page":42},"UniteLabs SDK","\u002Ftechnical-reference\u002Fpython-sdk","10.technical-reference\u002F20.python-sdk",[597,600],{"title":585,"path":598,"stem":599},"\u002Ftechnical-reference\u002Fpython-sdk\u002Finstall","10.technical-reference\u002F20.python-sdk\u002F10.install",{"title":589,"path":601,"stem":602},"\u002Ftechnical-reference\u002Fpython-sdk\u002Fchangelog","10.technical-reference\u002F20.python-sdk\u002F20.changelog",{"title":604,"path":605,"stem":606,"children":607,"page":42},"Liquid Handling SDK","\u002Ftechnical-reference\u002Fpython-liquid-handling-sdk","10.technical-reference\u002F30.python-liquid-handling-sdk",[608,611],{"title":585,"path":609,"stem":610},"\u002Ftechnical-reference\u002Fpython-liquid-handling-sdk\u002Finstall","10.technical-reference\u002F30.python-liquid-handling-sdk\u002F10.install",{"title":589,"path":612,"stem":613},"\u002Ftechnical-reference\u002Fpython-liquid-handling-sdk\u002Fchangelog","10.technical-reference\u002F30.python-liquid-handling-sdk\u002F20.changelog",{"title":163,"path":615,"stem":616,"children":617,"page":42},"\u002Ftechnical-reference\u002Flabware","10.technical-reference\u002F40.labware",[618,621],{"title":585,"path":619,"stem":620},"\u002Ftechnical-reference\u002Flabware\u002Finstall","10.technical-reference\u002F40.labware\u002F10.install",{"title":589,"path":622,"stem":623},"\u002Ftechnical-reference\u002Flabware\u002Fchangelog","10.technical-reference\u002F40.labware\u002F20.changelog",{"title":625,"path":626,"stem":627,"children":628,"page":42},"Python Omnibus","\u002Ftechnical-reference\u002Fpython-omnibus","10.technical-reference\u002F50.python-omnibus",[629,632],{"title":585,"path":630,"stem":631},"\u002Ftechnical-reference\u002Fpython-omnibus\u002Finstall","10.technical-reference\u002F50.python-omnibus\u002F10.install",{"title":589,"path":633,"stem":634},"\u002Ftechnical-reference\u002Fpython-omnibus\u002Fchangelog","10.technical-reference\u002F50.python-omnibus\u002F20.changelog",{"title":636,"path":637,"stem":638,"children":639,"page":42},"Python CDK","\u002Ftechnical-reference\u002Fpython-cdk","10.technical-reference\u002F60.python-cdk",[640,643],{"title":585,"path":641,"stem":642},"\u002Ftechnical-reference\u002Fpython-cdk\u002Finstall","10.technical-reference\u002F60.python-cdk\u002F10.install",{"title":589,"path":644,"stem":645},"\u002Ftechnical-reference\u002Fpython-cdk\u002Fchangelog","10.technical-reference\u002F60.python-cdk\u002F20.changelog",{"title":647,"path":648,"stem":649,"children":650},"Connector Development (CDK)","\u002Fconnector-development","20.connector-development",[651,653,672,705,713,744],{"title":647,"path":648,"stem":652},"20.connector-development\u002Findex",{"title":654,"path":655,"stem":656,"children":657,"page":42},"Getting Started","\u002Fconnector-development\u002Fgetting-started","20.connector-development\u002F10.getting-started",[658,661,665,668],{"title":536,"path":659,"stem":660},"\u002Fconnector-development\u002Fgetting-started\u002Foverview","20.connector-development\u002F10.getting-started\u002F10.overview",{"title":662,"path":663,"stem":664},"What's new","\u002Fconnector-development\u002Fgetting-started\u002Fwhats-new","20.connector-development\u002F10.getting-started\u002F20.whats-new",{"title":77,"path":666,"stem":667},"\u002Fconnector-development\u002Fgetting-started\u002Finstallation","20.connector-development\u002F10.getting-started\u002F30.installation",{"title":669,"path":670,"stem":671},"Contributing","\u002Fconnector-development\u002Fgetting-started\u002Fcontributing","20.connector-development\u002F10.getting-started\u002F40.contributing",{"title":673,"path":674,"stem":675,"children":676,"page":42},"Tutorial","\u002Fconnector-development\u002Ftutorial","20.connector-development\u002F20.tutorial",[677,681,685,689,693,697,701],{"title":678,"path":679,"stem":680},"Walkthrough","\u002Fconnector-development\u002Ftutorial\u002Fwalkthrough","20.connector-development\u002F20.tutorial\u002F00.walkthrough",{"title":682,"path":683,"stem":684},"SiLA Endpoints","\u002Fconnector-development\u002Ftutorial\u002Fsila-endpoints","20.connector-development\u002F20.tutorial\u002F10.sila-endpoints",{"title":686,"path":687,"stem":688},"Data Endpoints","\u002Fconnector-development\u002Ftutorial\u002Fdata-endpoints","20.connector-development\u002F20.tutorial\u002F20.data-endpoints",{"title":690,"path":691,"stem":692},"Controls","\u002Fconnector-development\u002Ftutorial\u002Fcontrols","20.connector-development\u002F20.tutorial\u002F30.controls",{"title":694,"path":695,"stem":696},"SiLA Data Types","\u002Fconnector-development\u002Ftutorial\u002Fdata-types","20.connector-development\u002F20.tutorial\u002F40.data-types",{"title":698,"path":699,"stem":700},"Feature","\u002Fconnector-development\u002Ftutorial\u002Fconnector-structure","20.connector-development\u002F20.tutorial\u002F70.connector-structure",{"title":702,"path":703,"stem":704},"Hardware Communication","\u002Fconnector-development\u002Ftutorial\u002Fhardware-communication","20.connector-development\u002F20.tutorial\u002F90.hardware-communication",{"title":94,"path":706,"stem":707,"children":708,"page":42},"\u002Fconnector-development\u002Fconcepts","20.connector-development\u002F30.concepts",[709],{"title":710,"path":711,"stem":712},"SiLA","\u002Fconnector-development\u002Fconcepts\u002Fsila","20.connector-development\u002F30.concepts\u002Fsila",{"title":187,"path":714,"stem":715,"children":716,"page":42},"\u002Fconnector-development\u002Fguides","20.connector-development\u002F40.guides",[717,721,725,729,733,736,740],{"title":718,"path":719,"stem":720},"Configuration","\u002Fconnector-development\u002Fguides\u002Fconfiguration","20.connector-development\u002F40.guides\u002F20.configuration",{"title":722,"path":723,"stem":724},"Deployment","\u002Fconnector-development\u002Fguides\u002Fdeployment","20.connector-development\u002F40.guides\u002F30.deployment",{"title":726,"path":727,"stem":728},"Security","\u002Fconnector-development\u002Fguides\u002Fsecurity","20.connector-development\u002F40.guides\u002F40.security",{"title":730,"path":731,"stem":732},"Subscriptions","\u002Fconnector-development\u002Fguides\u002Fsubscriptions","20.connector-development\u002F40.guides\u002F45.subscriptions",{"title":183,"path":734,"stem":735},"\u002Fconnector-development\u002Fguides\u002Ferror-handling","20.connector-development\u002F40.guides\u002F50.error-handling",{"title":737,"path":738,"stem":739},"Serial Troubleshooting","\u002Fconnector-development\u002Fguides\u002Fserial-troubleshooting","20.connector-development\u002F40.guides\u002F70.serial-troubleshooting",{"title":741,"path":742,"stem":743},"Testing","\u002Fconnector-development\u002Fguides\u002Ftesting","20.connector-development\u002F40.guides\u002F80.testing",{"title":745,"path":746,"stem":747,"children":748},"References","\u002Fconnector-development\u002Freference","20.connector-development\u002F50.reference\u002Findex",[749],{"title":745,"path":746,"stem":747},{"id":751,"title":564,"body":752,"description":3548,"extension":3549,"links":3550,"meta":3551,"navigation":915,"path":565,"seo":3552,"stem":566,"__hash__":3553},"docs\u002F08.observe\u002F30.guides\u002F20.building-an-etl.md",{"type":753,"value":754,"toc":3535},"minimark",[755,759,765,814,819,826,847,854,858,869,1171,1175,1186,1451,1455,1458,1652,1656,1659,2055,2059,2062,2209,2213,2219,3397,3401,3444,3448,3455,3459,3465,3497,3508,3512,3531],[756,757,758],"p",{},"This guide walks through building a production-ready ETL pipeline that monitors a LabChip GXII Touch instrument directory, archives raw files to Object Storage, and writes structured records to the Data Warehouse. The same pattern applies to any File System Connector and any instrument type.",[756,760,761],{},[762,763,764],"strong",{},"Prerequisites",[766,767,768,777,792],"ul",{},[769,770,771,772,776],"li",{},"A running File System Connector (e.g., ",[773,774,775],"code",{},"\"File System (LabChip demo)\"",").",[769,778,779,782,783,786,787,791],{},[773,780,781],{},"datalake-admin"," and ",[773,784,785],{},"warehouse-admin"," secrets configured in UniteLabs. See ",[788,789,548],"a",{"href":790},"\u002Fobserve\u002Fconcepts\u002Fsecrets\u002F",".",[769,793,794,795,798,799,798,802,798,805,798,808,798,811,791],{},"Python packages: ",[773,796,797],{},"prefect>=3.0",", ",[773,800,801],{},"unitelabs-sdk",[773,803,804],{},"sqlmodel",[773,806,807],{},"boto3",[773,809,810],{},"pandas",[773,812,813],{},"asyncpg",[815,816,818],"h2",{"id":817},"architecture","Architecture",[756,820,821,822,825],{},"The pipeline uses three sequential stages inside a single Prefect ",[773,823,824],{},"@flow",":",[827,828,829,835,841],"ol",{},[769,830,831,834],{},[762,832,833],{},"Reconcile"," — on startup, compare the connector's current file list against Warehouse records to detect files that were written while the pipeline was offline.",[769,836,837,840],{},[762,838,839],{},"Process backlog"," — archive and register every reconciled file that has not yet been stored.",[769,842,843,846],{},[762,844,845],{},"Watch in real time"," — subscribe to change events and process new files as they arrive.",[848,849,851],"callout",{"icon":850},"i-heroicons-light-bulb",[756,852,853],{},"This three-stage pattern ensures no files are skipped even if the workflow was offline when the instrument wrote results.",[815,855,857],{"id":856},"define-the-data-model","Define the data model",[756,859,860,861,864,865,868],{},"Define a ",[773,862,863],{},"SourceFiles"," model to track every file the pipeline has seen. Keep it in a shared ",[773,866,867],{},"models.py"," so both the flow and any query scripts can import it:",[870,871,876],"pre",{"className":872,"code":873,"filename":867,"language":874,"meta":875,"style":875},"language-python shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","from datetime import datetime\nfrom typing import Any\n\nimport sqlalchemy as sa\nfrom sqlmodel import Field, SQLModel\n\n\nclass SourceFiles(SQLModel, table=True):\n    __tablename__ = \"source_files\"\n\n    id: int | None = Field(default=None, primary_key=True)\n    file_path: str                          # Original path on the connector\n    s3_object_path: str                     # Path inside the S3 bucket\n    connector_name: str\n    instrument_type: str\n    file_metadata: dict[str, Any] | None = Field(\n        default=None, sa_type=sa.JSON\n    )\n    updated_at: datetime\n","python","",[773,877,878,897,910,917,931,951,956,961,987,1006,1011,1048,1063,1076,1087,1097,1133,1155,1161],{"__ignoreMap":875},[879,880,883,887,891,894],"span",{"class":881,"line":882},"line",1,[879,884,886],{"class":885},"s7zQu","from",[879,888,890],{"class":889},"sTEyZ"," datetime ",[879,892,893],{"class":885},"import",[879,895,896],{"class":889}," datetime\n",[879,898,900,902,905,907],{"class":881,"line":899},2,[879,901,886],{"class":885},[879,903,904],{"class":889}," typing ",[879,906,893],{"class":885},[879,908,909],{"class":889}," Any\n",[879,911,913],{"class":881,"line":912},3,[879,914,916],{"emptyLinePlaceholder":915},true,"\n",[879,918,920,922,925,928],{"class":881,"line":919},4,[879,921,893],{"class":885},[879,923,924],{"class":889}," sqlalchemy ",[879,926,927],{"class":885},"as",[879,929,930],{"class":889}," sa\n",[879,932,934,936,939,941,944,948],{"class":881,"line":933},5,[879,935,886],{"class":885},[879,937,938],{"class":889}," sqlmodel ",[879,940,893],{"class":885},[879,942,943],{"class":889}," Field",[879,945,947],{"class":946},"sMK4o",",",[879,949,950],{"class":889}," SQLModel\n",[879,952,954],{"class":881,"line":953},6,[879,955,916],{"emptyLinePlaceholder":915},[879,957,959],{"class":881,"line":958},7,[879,960,916],{"emptyLinePlaceholder":915},[879,962,964,968,972,975,978,980,984],{"class":881,"line":963},8,[879,965,967],{"class":966},"spNyl","class",[879,969,971],{"class":970},"sBMFI"," SourceFiles",[879,973,974],{"class":946},"(",[879,976,977],{"class":970},"SQLModel",[879,979,947],{"class":946},[879,981,983],{"class":982},"sHdIc"," table",[879,985,986],{"class":946},"=True):\n",[879,988,990,993,996,999,1003],{"class":881,"line":989},9,[879,991,992],{"class":889},"    __tablename__ ",[879,994,995],{"class":946},"=",[879,997,998],{"class":946}," \"",[879,1000,1002],{"class":1001},"sfazB","source_files",[879,1004,1005],{"class":946},"\"\n",[879,1007,1009],{"class":881,"line":1008},10,[879,1010,916],{"emptyLinePlaceholder":915},[879,1012,1014,1018,1020,1023,1026,1029,1032,1034,1036,1039,1042,1045],{"class":881,"line":1013},11,[879,1015,1017],{"class":1016},"s2Zo4","    id",[879,1019,825],{"class":946},[879,1021,1022],{"class":970}," int",[879,1024,1025],{"class":946}," |",[879,1027,1028],{"class":946}," None",[879,1030,1031],{"class":946}," =",[879,1033,943],{"class":1016},[879,1035,974],{"class":946},[879,1037,1038],{"class":982},"default",[879,1040,1041],{"class":946},"=None,",[879,1043,1044],{"class":982}," primary_key",[879,1046,1047],{"class":946},"=True)\n",[879,1049,1051,1054,1056,1059],{"class":881,"line":1050},12,[879,1052,1053],{"class":889},"    file_path",[879,1055,825],{"class":946},[879,1057,1058],{"class":970}," str",[879,1060,1062],{"class":1061},"sHwdD","                          # Original path on the connector\n",[879,1064,1066,1069,1071,1073],{"class":881,"line":1065},13,[879,1067,1068],{"class":889},"    s3_object_path",[879,1070,825],{"class":946},[879,1072,1058],{"class":970},[879,1074,1075],{"class":1061},"                     # Path inside the S3 bucket\n",[879,1077,1079,1082,1084],{"class":881,"line":1078},14,[879,1080,1081],{"class":889},"    connector_name",[879,1083,825],{"class":946},[879,1085,1086],{"class":970}," str\n",[879,1088,1090,1093,1095],{"class":881,"line":1089},15,[879,1091,1092],{"class":889},"    instrument_type",[879,1094,825],{"class":946},[879,1096,1086],{"class":970},[879,1098,1100,1103,1105,1108,1111,1114,1116,1119,1122,1124,1126,1128,1130],{"class":881,"line":1099},16,[879,1101,1102],{"class":889},"    file_metadata",[879,1104,825],{"class":946},[879,1106,1107],{"class":889}," dict",[879,1109,1110],{"class":946},"[",[879,1112,1113],{"class":970},"str",[879,1115,947],{"class":946},[879,1117,1118],{"class":889}," Any",[879,1120,1121],{"class":946},"]",[879,1123,1025],{"class":946},[879,1125,1028],{"class":946},[879,1127,1031],{"class":946},[879,1129,943],{"class":1016},[879,1131,1132],{"class":946},"(\n",[879,1134,1136,1139,1141,1144,1146,1149,1151],{"class":881,"line":1135},17,[879,1137,1138],{"class":982},"        default",[879,1140,1041],{"class":946},[879,1142,1143],{"class":982}," sa_type",[879,1145,995],{"class":946},[879,1147,1148],{"class":1016},"sa",[879,1150,791],{"class":946},[879,1152,1154],{"class":1153},"swJcz","JSON\n",[879,1156,1158],{"class":881,"line":1157},18,[879,1159,1160],{"class":946},"    )\n",[879,1162,1164,1167,1169],{"class":881,"line":1163},19,[879,1165,1166],{"class":889},"    updated_at",[879,1168,825],{"class":946},[879,1170,896],{"class":889},[815,1172,1174],{"id":1173},"task-register-a-source-file","Task: register a source file",[756,1176,1177,1178,1181,1182,1185],{},"Write a single file record to the Warehouse. Using ",[773,1179,1180],{},"upsert_record"," with ",[773,1183,1184],{},"file_path"," as the conflict key makes the task safe to retry:",[870,1187,1189],{"className":872,"code":1188,"language":874,"meta":875,"style":875},"from datetime import datetime, timezone\nfrom prefect import task\n\n\n@task(name=\"Register Source File\")\nasync def register_source_file_task(\n    file_path: str,\n    s3_object_path: str,\n    connector_name: str,\n    instrument_type: str,\n    warehouse,\n) -> SourceFiles:\n    record = SourceFiles(\n        file_path=file_path,\n        s3_object_path=s3_object_path,\n        connector_name=connector_name,\n        instrument_type=instrument_type,\n        updated_at=datetime.now(tz=timezone.utc),\n    )\n    await warehouse.upsert_record(record)\n    return record\n",[773,1190,1191,1207,1219,1223,1227,1253,1266,1277,1287,1297,1307,1314,1327,1338,1349,1361,1373,1385,1418,1422,1442],{"__ignoreMap":875},[879,1192,1193,1195,1197,1199,1202,1204],{"class":881,"line":882},[879,1194,886],{"class":885},[879,1196,890],{"class":889},[879,1198,893],{"class":885},[879,1200,1201],{"class":889}," datetime",[879,1203,947],{"class":946},[879,1205,1206],{"class":889}," timezone\n",[879,1208,1209,1211,1214,1216],{"class":881,"line":899},[879,1210,886],{"class":885},[879,1212,1213],{"class":889}," prefect ",[879,1215,893],{"class":885},[879,1217,1218],{"class":889}," task\n",[879,1220,1221],{"class":881,"line":912},[879,1222,916],{"emptyLinePlaceholder":915},[879,1224,1225],{"class":881,"line":919},[879,1226,916],{"emptyLinePlaceholder":915},[879,1228,1229,1232,1235,1237,1240,1242,1245,1248,1250],{"class":881,"line":933},[879,1230,1231],{"class":946},"@",[879,1233,1234],{"class":1016},"task",[879,1236,974],{"class":946},[879,1238,1239],{"class":982},"name",[879,1241,995],{"class":946},[879,1243,1244],{"class":946},"\"",[879,1246,1247],{"class":1001},"Register Source File",[879,1249,1244],{"class":946},[879,1251,1252],{"class":946},")\n",[879,1254,1255,1258,1261,1264],{"class":881,"line":953},[879,1256,1257],{"class":966},"async",[879,1259,1260],{"class":966}," def",[879,1262,1263],{"class":1016}," register_source_file_task",[879,1265,1132],{"class":946},[879,1267,1268,1270,1272,1274],{"class":881,"line":958},[879,1269,1053],{"class":982},[879,1271,825],{"class":946},[879,1273,1058],{"class":970},[879,1275,1276],{"class":946},",\n",[879,1278,1279,1281,1283,1285],{"class":881,"line":963},[879,1280,1068],{"class":982},[879,1282,825],{"class":946},[879,1284,1058],{"class":970},[879,1286,1276],{"class":946},[879,1288,1289,1291,1293,1295],{"class":881,"line":989},[879,1290,1081],{"class":982},[879,1292,825],{"class":946},[879,1294,1058],{"class":970},[879,1296,1276],{"class":946},[879,1298,1299,1301,1303,1305],{"class":881,"line":1008},[879,1300,1092],{"class":982},[879,1302,825],{"class":946},[879,1304,1058],{"class":970},[879,1306,1276],{"class":946},[879,1308,1309,1312],{"class":881,"line":1013},[879,1310,1311],{"class":982},"    warehouse",[879,1313,1276],{"class":946},[879,1315,1316,1319,1322,1324],{"class":881,"line":1050},[879,1317,1318],{"class":946},")",[879,1320,1321],{"class":946}," ->",[879,1323,971],{"class":889},[879,1325,1326],{"class":946},":\n",[879,1328,1329,1332,1334,1336],{"class":881,"line":1065},[879,1330,1331],{"class":889},"    record ",[879,1333,995],{"class":946},[879,1335,971],{"class":1016},[879,1337,1132],{"class":946},[879,1339,1340,1343,1345,1347],{"class":881,"line":1078},[879,1341,1342],{"class":982},"        file_path",[879,1344,995],{"class":946},[879,1346,1184],{"class":1016},[879,1348,1276],{"class":946},[879,1350,1351,1354,1356,1359],{"class":881,"line":1089},[879,1352,1353],{"class":982},"        s3_object_path",[879,1355,995],{"class":946},[879,1357,1358],{"class":1016},"s3_object_path",[879,1360,1276],{"class":946},[879,1362,1363,1366,1368,1371],{"class":881,"line":1099},[879,1364,1365],{"class":982},"        connector_name",[879,1367,995],{"class":946},[879,1369,1370],{"class":1016},"connector_name",[879,1372,1276],{"class":946},[879,1374,1375,1378,1380,1383],{"class":881,"line":1135},[879,1376,1377],{"class":982},"        instrument_type",[879,1379,995],{"class":946},[879,1381,1382],{"class":1016},"instrument_type",[879,1384,1276],{"class":946},[879,1386,1387,1390,1392,1395,1397,1400,1402,1405,1407,1410,1412,1415],{"class":881,"line":1157},[879,1388,1389],{"class":982},"        updated_at",[879,1391,995],{"class":946},[879,1393,1394],{"class":1016},"datetime",[879,1396,791],{"class":946},[879,1398,1399],{"class":1016},"now",[879,1401,974],{"class":946},[879,1403,1404],{"class":982},"tz",[879,1406,995],{"class":946},[879,1408,1409],{"class":1016},"timezone",[879,1411,791],{"class":946},[879,1413,1414],{"class":1153},"utc",[879,1416,1417],{"class":946},"),\n",[879,1419,1420],{"class":881,"line":1163},[879,1421,1160],{"class":946},[879,1423,1425,1428,1431,1433,1435,1437,1440],{"class":881,"line":1424},20,[879,1426,1427],{"class":885},"    await",[879,1429,1430],{"class":889}," warehouse",[879,1432,791],{"class":946},[879,1434,1180],{"class":1016},[879,1436,974],{"class":946},[879,1438,1439],{"class":1016},"record",[879,1441,1252],{"class":946},[879,1443,1445,1448],{"class":881,"line":1444},21,[879,1446,1447],{"class":885},"    return",[879,1449,1450],{"class":889}," record\n",[815,1452,1454],{"id":1453},"task-archive-to-object-storage","Task: archive to Object Storage",[756,1456,1457],{},"Upload the raw file from the connector directly to S3. The transfer happens on the connector side — no data passes through the workflow host:",[870,1459,1461],{"className":872,"code":1460,"language":874,"meta":875,"style":875},"from unitelabs.sdk import AsyncApiClient\n\n\n@task(name=\"Archive to Object Storage\")\nasync def archive_file_task(\n    connector_name: str,\n    file_path: str,\n    s3_url: str,\n) -> str:\n    async with AsyncApiClient() as client:\n        device = await client.get_service_by_name(connector_name)\n        await device.s3_service.upload_file(path=file_path, s3_url=s3_url)\n    return s3_url\n",[773,1462,1463,1480,1484,1488,1509,1520,1530,1540,1551,1561,1583,1606,1645],{"__ignoreMap":875},[879,1464,1465,1467,1470,1472,1475,1477],{"class":881,"line":882},[879,1466,886],{"class":885},[879,1468,1469],{"class":889}," unitelabs",[879,1471,791],{"class":946},[879,1473,1474],{"class":889},"sdk ",[879,1476,893],{"class":885},[879,1478,1479],{"class":889}," AsyncApiClient\n",[879,1481,1482],{"class":881,"line":899},[879,1483,916],{"emptyLinePlaceholder":915},[879,1485,1486],{"class":881,"line":912},[879,1487,916],{"emptyLinePlaceholder":915},[879,1489,1490,1492,1494,1496,1498,1500,1502,1505,1507],{"class":881,"line":919},[879,1491,1231],{"class":946},[879,1493,1234],{"class":1016},[879,1495,974],{"class":946},[879,1497,1239],{"class":982},[879,1499,995],{"class":946},[879,1501,1244],{"class":946},[879,1503,1504],{"class":1001},"Archive to Object Storage",[879,1506,1244],{"class":946},[879,1508,1252],{"class":946},[879,1510,1511,1513,1515,1518],{"class":881,"line":933},[879,1512,1257],{"class":966},[879,1514,1260],{"class":966},[879,1516,1517],{"class":1016}," archive_file_task",[879,1519,1132],{"class":946},[879,1521,1522,1524,1526,1528],{"class":881,"line":953},[879,1523,1081],{"class":982},[879,1525,825],{"class":946},[879,1527,1058],{"class":970},[879,1529,1276],{"class":946},[879,1531,1532,1534,1536,1538],{"class":881,"line":958},[879,1533,1053],{"class":982},[879,1535,825],{"class":946},[879,1537,1058],{"class":970},[879,1539,1276],{"class":946},[879,1541,1542,1545,1547,1549],{"class":881,"line":963},[879,1543,1544],{"class":982},"    s3_url",[879,1546,825],{"class":946},[879,1548,1058],{"class":970},[879,1550,1276],{"class":946},[879,1552,1553,1555,1557,1559],{"class":881,"line":989},[879,1554,1318],{"class":946},[879,1556,1321],{"class":946},[879,1558,1058],{"class":970},[879,1560,1326],{"class":946},[879,1562,1563,1566,1569,1572,1575,1578,1581],{"class":881,"line":1008},[879,1564,1565],{"class":885},"    async",[879,1567,1568],{"class":885}," with",[879,1570,1571],{"class":1016}," AsyncApiClient",[879,1573,1574],{"class":946},"()",[879,1576,1577],{"class":885}," as",[879,1579,1580],{"class":889}," client",[879,1582,1326],{"class":946},[879,1584,1585,1588,1590,1593,1595,1597,1600,1602,1604],{"class":881,"line":1013},[879,1586,1587],{"class":889},"        device ",[879,1589,995],{"class":946},[879,1591,1592],{"class":885}," await",[879,1594,1580],{"class":889},[879,1596,791],{"class":946},[879,1598,1599],{"class":1016},"get_service_by_name",[879,1601,974],{"class":946},[879,1603,1370],{"class":1016},[879,1605,1252],{"class":946},[879,1607,1608,1611,1614,1616,1619,1621,1624,1626,1629,1631,1633,1635,1638,1640,1643],{"class":881,"line":1050},[879,1609,1610],{"class":885},"        await",[879,1612,1613],{"class":889}," device",[879,1615,791],{"class":946},[879,1617,1618],{"class":1153},"s3_service",[879,1620,791],{"class":946},[879,1622,1623],{"class":1016},"upload_file",[879,1625,974],{"class":946},[879,1627,1628],{"class":982},"path",[879,1630,995],{"class":946},[879,1632,1184],{"class":1016},[879,1634,947],{"class":946},[879,1636,1637],{"class":982}," s3_url",[879,1639,995],{"class":946},[879,1641,1642],{"class":1016},"s3_url",[879,1644,1252],{"class":946},[879,1646,1647,1649],{"class":881,"line":1065},[879,1648,1447],{"class":885},[879,1650,1651],{"class":889}," s3_url\n",[815,1653,1655],{"id":1654},"task-reconcile-missed-files","Task: reconcile missed files",[756,1657,1658],{},"On startup, list all files currently visible to the connector and cross-reference against Warehouse records. Returns only files not yet registered:",[870,1660,1662],{"className":872,"code":1661,"language":874,"meta":875,"style":875},"@task(name=\"Reconcile Connector Files\")\nasync def reconcile_connector_files_task(\n    connector_name: str,\n    instrument_type: str,\n    warehouse,\n) -> dict:\n    async with AsyncApiClient() as client:\n        device = await client.get_service_by_name(connector_name)\n        whitelist = await device.folder_service.get_folder_whitelist()\n\n        all_files: list[str] = []\n        for folder in whitelist:\n            files = await device.folder_service.get_folder_contents(path=folder)\n            all_files.extend(files)\n\n    existing = await warehouse.query_records(\n        SourceFiles,\n        instrument_type=instrument_type,\n    )\n    existing_paths = {r.file_path for r in existing}\n\n    reconciled = [f for f in all_files if f not in existing_paths]\n    return {\n        \"reconciled_files\": reconciled,\n        \"total_on_disk\": len(all_files),\n    }\n",[773,1663,1664,1685,1696,1706,1716,1722,1732,1748,1768,1792,1796,1817,1833,1864,1881,1885,1903,1910,1920,1924,1955,1959,2001,2009,2027,2049],{"__ignoreMap":875},[879,1665,1666,1668,1670,1672,1674,1676,1678,1681,1683],{"class":881,"line":882},[879,1667,1231],{"class":946},[879,1669,1234],{"class":1016},[879,1671,974],{"class":946},[879,1673,1239],{"class":982},[879,1675,995],{"class":946},[879,1677,1244],{"class":946},[879,1679,1680],{"class":1001},"Reconcile Connector Files",[879,1682,1244],{"class":946},[879,1684,1252],{"class":946},[879,1686,1687,1689,1691,1694],{"class":881,"line":899},[879,1688,1257],{"class":966},[879,1690,1260],{"class":966},[879,1692,1693],{"class":1016}," reconcile_connector_files_task",[879,1695,1132],{"class":946},[879,1697,1698,1700,1702,1704],{"class":881,"line":912},[879,1699,1081],{"class":982},[879,1701,825],{"class":946},[879,1703,1058],{"class":970},[879,1705,1276],{"class":946},[879,1707,1708,1710,1712,1714],{"class":881,"line":919},[879,1709,1092],{"class":982},[879,1711,825],{"class":946},[879,1713,1058],{"class":970},[879,1715,1276],{"class":946},[879,1717,1718,1720],{"class":881,"line":933},[879,1719,1311],{"class":982},[879,1721,1276],{"class":946},[879,1723,1724,1726,1728,1730],{"class":881,"line":953},[879,1725,1318],{"class":946},[879,1727,1321],{"class":946},[879,1729,1107],{"class":970},[879,1731,1326],{"class":946},[879,1733,1734,1736,1738,1740,1742,1744,1746],{"class":881,"line":958},[879,1735,1565],{"class":885},[879,1737,1568],{"class":885},[879,1739,1571],{"class":1016},[879,1741,1574],{"class":946},[879,1743,1577],{"class":885},[879,1745,1580],{"class":889},[879,1747,1326],{"class":946},[879,1749,1750,1752,1754,1756,1758,1760,1762,1764,1766],{"class":881,"line":963},[879,1751,1587],{"class":889},[879,1753,995],{"class":946},[879,1755,1592],{"class":885},[879,1757,1580],{"class":889},[879,1759,791],{"class":946},[879,1761,1599],{"class":1016},[879,1763,974],{"class":946},[879,1765,1370],{"class":1016},[879,1767,1252],{"class":946},[879,1769,1770,1773,1775,1777,1779,1781,1784,1786,1789],{"class":881,"line":989},[879,1771,1772],{"class":889},"        whitelist ",[879,1774,995],{"class":946},[879,1776,1592],{"class":885},[879,1778,1613],{"class":889},[879,1780,791],{"class":946},[879,1782,1783],{"class":1153},"folder_service",[879,1785,791],{"class":946},[879,1787,1788],{"class":1016},"get_folder_whitelist",[879,1790,1791],{"class":946},"()\n",[879,1793,1794],{"class":881,"line":1008},[879,1795,916],{"emptyLinePlaceholder":915},[879,1797,1798,1801,1803,1806,1808,1810,1812,1814],{"class":881,"line":1013},[879,1799,1800],{"class":889},"        all_files",[879,1802,825],{"class":946},[879,1804,1805],{"class":889}," list",[879,1807,1110],{"class":946},[879,1809,1113],{"class":970},[879,1811,1121],{"class":946},[879,1813,1031],{"class":946},[879,1815,1816],{"class":946}," []\n",[879,1818,1819,1822,1825,1828,1831],{"class":881,"line":1050},[879,1820,1821],{"class":885},"        for",[879,1823,1824],{"class":889}," folder ",[879,1826,1827],{"class":885},"in",[879,1829,1830],{"class":889}," whitelist",[879,1832,1326],{"class":946},[879,1834,1835,1838,1840,1842,1844,1846,1848,1850,1853,1855,1857,1859,1862],{"class":881,"line":1065},[879,1836,1837],{"class":889},"            files ",[879,1839,995],{"class":946},[879,1841,1592],{"class":885},[879,1843,1613],{"class":889},[879,1845,791],{"class":946},[879,1847,1783],{"class":1153},[879,1849,791],{"class":946},[879,1851,1852],{"class":1016},"get_folder_contents",[879,1854,974],{"class":946},[879,1856,1628],{"class":982},[879,1858,995],{"class":946},[879,1860,1861],{"class":1016},"folder",[879,1863,1252],{"class":946},[879,1865,1866,1869,1871,1874,1876,1879],{"class":881,"line":1078},[879,1867,1868],{"class":889},"            all_files",[879,1870,791],{"class":946},[879,1872,1873],{"class":1016},"extend",[879,1875,974],{"class":946},[879,1877,1878],{"class":1016},"files",[879,1880,1252],{"class":946},[879,1882,1883],{"class":881,"line":1089},[879,1884,916],{"emptyLinePlaceholder":915},[879,1886,1887,1890,1892,1894,1896,1898,1901],{"class":881,"line":1099},[879,1888,1889],{"class":889},"    existing ",[879,1891,995],{"class":946},[879,1893,1592],{"class":885},[879,1895,1430],{"class":889},[879,1897,791],{"class":946},[879,1899,1900],{"class":1016},"query_records",[879,1902,1132],{"class":946},[879,1904,1905,1908],{"class":881,"line":1135},[879,1906,1907],{"class":1016},"        SourceFiles",[879,1909,1276],{"class":946},[879,1911,1912,1914,1916,1918],{"class":881,"line":1157},[879,1913,1377],{"class":982},[879,1915,995],{"class":946},[879,1917,1382],{"class":1016},[879,1919,1276],{"class":946},[879,1921,1922],{"class":881,"line":1163},[879,1923,1160],{"class":946},[879,1925,1926,1929,1931,1934,1937,1939,1941,1944,1947,1949,1952],{"class":881,"line":1424},[879,1927,1928],{"class":889},"    existing_paths ",[879,1930,995],{"class":946},[879,1932,1933],{"class":946}," {",[879,1935,1936],{"class":889},"r",[879,1938,791],{"class":946},[879,1940,1184],{"class":1153},[879,1942,1943],{"class":885}," for",[879,1945,1946],{"class":889}," r ",[879,1948,1827],{"class":885},[879,1950,1951],{"class":889}," existing",[879,1953,1954],{"class":946},"}\n",[879,1956,1957],{"class":881,"line":1444},[879,1958,916],{"emptyLinePlaceholder":915},[879,1960,1962,1965,1967,1970,1973,1976,1979,1981,1984,1987,1989,1992,1995,1998],{"class":881,"line":1961},22,[879,1963,1964],{"class":889},"    reconciled ",[879,1966,995],{"class":946},[879,1968,1969],{"class":946}," [",[879,1971,1972],{"class":889},"f ",[879,1974,1975],{"class":885},"for",[879,1977,1978],{"class":889}," f ",[879,1980,1827],{"class":885},[879,1982,1983],{"class":889}," all_files ",[879,1985,1986],{"class":885},"if",[879,1988,1978],{"class":889},[879,1990,1991],{"class":946},"not",[879,1993,1994],{"class":946}," in",[879,1996,1997],{"class":889}," existing_paths",[879,1999,2000],{"class":946},"]\n",[879,2002,2004,2006],{"class":881,"line":2003},23,[879,2005,1447],{"class":885},[879,2007,2008],{"class":946}," {\n",[879,2010,2012,2015,2018,2020,2022,2025],{"class":881,"line":2011},24,[879,2013,2014],{"class":946},"        \"",[879,2016,2017],{"class":1001},"reconciled_files",[879,2019,1244],{"class":946},[879,2021,825],{"class":946},[879,2023,2024],{"class":889}," reconciled",[879,2026,1276],{"class":946},[879,2028,2030,2032,2035,2037,2039,2042,2044,2047],{"class":881,"line":2029},25,[879,2031,2014],{"class":946},[879,2033,2034],{"class":1001},"total_on_disk",[879,2036,1244],{"class":946},[879,2038,825],{"class":946},[879,2040,2041],{"class":1016}," len",[879,2043,974],{"class":946},[879,2045,2046],{"class":1016},"all_files",[879,2048,1417],{"class":946},[879,2050,2052],{"class":881,"line":2051},26,[879,2053,2054],{"class":946},"    }\n",[815,2056,2058],{"id":2057},"task-watch-for-new-files","Task: watch for new files",[756,2060,2061],{},"A long-running task that yields each new file event as it arrives from the connector:",[870,2063,2065],{"className":872,"code":2064,"language":874,"meta":875,"style":875},"@task(name=\"Watch for New Files\")\nasync def watch_for_new_files_task(connector_name: str):\n    async with AsyncApiClient() as client:\n        device = await client.get_service_by_name(connector_name)\n        async with await device.folder_service.subscribe_changes() as sub:\n            async for file_path in sub:\n                yield {\"file_path\": file_path}\n",[773,2066,2067,2088,2108,2124,2144,2173,2189],{"__ignoreMap":875},[879,2068,2069,2071,2073,2075,2077,2079,2081,2084,2086],{"class":881,"line":882},[879,2070,1231],{"class":946},[879,2072,1234],{"class":1016},[879,2074,974],{"class":946},[879,2076,1239],{"class":982},[879,2078,995],{"class":946},[879,2080,1244],{"class":946},[879,2082,2083],{"class":1001},"Watch for New Files",[879,2085,1244],{"class":946},[879,2087,1252],{"class":946},[879,2089,2090,2092,2094,2097,2099,2101,2103,2105],{"class":881,"line":899},[879,2091,1257],{"class":966},[879,2093,1260],{"class":966},[879,2095,2096],{"class":1016}," watch_for_new_files_task",[879,2098,974],{"class":946},[879,2100,1370],{"class":982},[879,2102,825],{"class":946},[879,2104,1058],{"class":970},[879,2106,2107],{"class":946},"):\n",[879,2109,2110,2112,2114,2116,2118,2120,2122],{"class":881,"line":912},[879,2111,1565],{"class":885},[879,2113,1568],{"class":885},[879,2115,1571],{"class":1016},[879,2117,1574],{"class":946},[879,2119,1577],{"class":885},[879,2121,1580],{"class":889},[879,2123,1326],{"class":946},[879,2125,2126,2128,2130,2132,2134,2136,2138,2140,2142],{"class":881,"line":919},[879,2127,1587],{"class":889},[879,2129,995],{"class":946},[879,2131,1592],{"class":885},[879,2133,1580],{"class":889},[879,2135,791],{"class":946},[879,2137,1599],{"class":1016},[879,2139,974],{"class":946},[879,2141,1370],{"class":1016},[879,2143,1252],{"class":946},[879,2145,2146,2149,2151,2153,2155,2157,2159,2161,2164,2166,2168,2171],{"class":881,"line":933},[879,2147,2148],{"class":885},"        async",[879,2150,1568],{"class":885},[879,2152,1592],{"class":885},[879,2154,1613],{"class":889},[879,2156,791],{"class":946},[879,2158,1783],{"class":1153},[879,2160,791],{"class":946},[879,2162,2163],{"class":1016},"subscribe_changes",[879,2165,1574],{"class":946},[879,2167,1577],{"class":885},[879,2169,2170],{"class":889}," sub",[879,2172,1326],{"class":946},[879,2174,2175,2178,2180,2183,2185,2187],{"class":881,"line":953},[879,2176,2177],{"class":885},"            async",[879,2179,1943],{"class":885},[879,2181,2182],{"class":889}," file_path ",[879,2184,1827],{"class":885},[879,2186,2170],{"class":889},[879,2188,1326],{"class":946},[879,2190,2191,2194,2196,2198,2200,2202,2204,2207],{"class":881,"line":958},[879,2192,2193],{"class":885},"                yield",[879,2195,1933],{"class":946},[879,2197,1244],{"class":946},[879,2199,1184],{"class":1001},[879,2201,1244],{"class":946},[879,2203,825],{"class":946},[879,2205,2206],{"class":889}," file_path",[879,2208,1954],{"class":946},[815,2210,2212],{"id":2211},"the-flow-wiring-it-together","The flow: wiring it together",[756,2214,2215,2216,2218],{},"The ",[773,2217,824],{}," function runs all three stages in order. Stages 2 and 3 call the same archive + register tasks so the logic is never duplicated:",[870,2220,2222],{"className":872,"code":2221,"language":874,"meta":875,"style":875},"from unitelabs.sdk import AsyncApiClient\nfrom prefect import flow, get_run_logger\n\n\n@flow(log_prints=True, name=\"LabChip GXII Touch ETL Pipeline\")\nasync def labchip_etl_flow(\n    connector_name: str = \"File System (Labchip demo)\",\n    instrument_type: str = \"labchip\",\n    s3_bucket_prefix: str = \"data\u002Finstruments\u002Flabchip\",\n) -> None:\n    logger = get_run_logger()\n\n    # Load S3 credentials from secrets\n    async with AsyncApiClient() as client:\n        secrets = await client.get(\"\u002Fsecrets\")\n    datalake = next(s for s in secrets if s[\"name\"] == \"datalake-admin\")\n    p = datalake[\"parameters\"]\n    s3_base = (\n        f\"s3s:\u002F\u002F{p['minio_root_user']}:{p['minio_root_password']}\"\n        f\"@{p['aws_client_parameters']['endpoint_url'].replace('http:\u002F\u002F', '')}\"\n        f\"\u002F{p['aws_client_parameters']['bucket_name']}\u002F{s3_bucket_prefix}\"\n    )\n\n    warehouse = get_warehouse_client()\n    await warehouse.ensure_tables_exist()\n\n    # ── Stage 1: Reconcile files missed while offline ──────────────────────\n    logger.info(\"Stage 1: Reconciling missed files\")\n    reconciliation = await reconcile_connector_files_task(\n        connector_name=connector_name,\n        instrument_type=instrument_type,\n        warehouse=warehouse,\n    )\n    logger.info(\n        f\"Found {len(reconciliation['reconciled_files'])} unregistered files \"\n        f\"out of {reconciliation['total_on_disk']} on disk\"\n    )\n\n    # ── Stage 2: Process the backlog ───────────────────────────────────────\n    for file_path in reconciliation[\"reconciled_files\"]:\n        filename = file_path.split(\"\u002F\")[-1]\n        s3_url = f\"{s3_base}\u002F{filename}\"\n\n        await archive_file_task(\n            connector_name=connector_name,\n            file_path=file_path,\n            s3_url=s3_url,\n        )\n        await register_source_file_task(\n            file_path=file_path,\n            s3_object_path=f\"{s3_bucket_prefix}\u002F{filename}\",\n            connector_name=connector_name,\n            instrument_type=instrument_type,\n            warehouse=warehouse,\n        )\n\n    # ── Stage 3: Real-time monitoring ──────────────────────────────────────\n    logger.info(\"Stage 3: Watching for new files in real time\")\n    async for change in watch_for_new_files_task(connector_name):\n        file_path = change[\"file_path\"]\n        filename = file_path.split(\"\u002F\")[-1]\n        s3_url = f\"{s3_base}\u002F{filename}\"\n\n        logger.info(f\"New file detected: {file_path}\")\n        await archive_file_task(\n            connector_name=connector_name,\n            file_path=file_path,\n            s3_url=s3_url,\n        )\n        await register_source_file_task(\n            file_path=file_path,\n            s3_object_path=f\"{s3_bucket_prefix}\u002F{filename}\",\n            connector_name=connector_name,\n            instrument_type=instrument_type,\n            warehouse=warehouse,\n        )\n",[773,2223,2224,2238,2254,2258,2262,2291,2302,2321,2340,2360,2369,2381,2385,2390,2406,2433,2484,2505,2515,2565,2621,2665,2669,2673,2685,2698,2702,2708,2730,2744,2755,2766,2779,2784,2795,2829,2856,2861,2866,2872,2896,2927,2958,2963,2972,2984,2996,3008,3014,3023,3034,3065,3076,3088,3100,3105,3110,3116,3136,3156,3177,3204,3231,3236,3263,3272,3283,3294,3305,3310,3319,3330,3359,3370,3381,3392],{"__ignoreMap":875},[879,2225,2226,2228,2230,2232,2234,2236],{"class":881,"line":882},[879,2227,886],{"class":885},[879,2229,1469],{"class":889},[879,2231,791],{"class":946},[879,2233,1474],{"class":889},[879,2235,893],{"class":885},[879,2237,1479],{"class":889},[879,2239,2240,2242,2244,2246,2249,2251],{"class":881,"line":899},[879,2241,886],{"class":885},[879,2243,1213],{"class":889},[879,2245,893],{"class":885},[879,2247,2248],{"class":889}," flow",[879,2250,947],{"class":946},[879,2252,2253],{"class":889}," get_run_logger\n",[879,2255,2256],{"class":881,"line":912},[879,2257,916],{"emptyLinePlaceholder":915},[879,2259,2260],{"class":881,"line":919},[879,2261,916],{"emptyLinePlaceholder":915},[879,2263,2264,2266,2269,2271,2274,2277,2280,2282,2284,2287,2289],{"class":881,"line":933},[879,2265,1231],{"class":946},[879,2267,2268],{"class":1016},"flow",[879,2270,974],{"class":946},[879,2272,2273],{"class":982},"log_prints",[879,2275,2276],{"class":946},"=True,",[879,2278,2279],{"class":982}," name",[879,2281,995],{"class":946},[879,2283,1244],{"class":946},[879,2285,2286],{"class":1001},"LabChip GXII Touch ETL Pipeline",[879,2288,1244],{"class":946},[879,2290,1252],{"class":946},[879,2292,2293,2295,2297,2300],{"class":881,"line":953},[879,2294,1257],{"class":966},[879,2296,1260],{"class":966},[879,2298,2299],{"class":1016}," labchip_etl_flow",[879,2301,1132],{"class":946},[879,2303,2304,2306,2308,2310,2312,2314,2317,2319],{"class":881,"line":958},[879,2305,1081],{"class":982},[879,2307,825],{"class":946},[879,2309,1058],{"class":970},[879,2311,1031],{"class":946},[879,2313,998],{"class":946},[879,2315,2316],{"class":1001},"File System (Labchip demo)",[879,2318,1244],{"class":946},[879,2320,1276],{"class":946},[879,2322,2323,2325,2327,2329,2331,2333,2336,2338],{"class":881,"line":963},[879,2324,1092],{"class":982},[879,2326,825],{"class":946},[879,2328,1058],{"class":970},[879,2330,1031],{"class":946},[879,2332,998],{"class":946},[879,2334,2335],{"class":1001},"labchip",[879,2337,1244],{"class":946},[879,2339,1276],{"class":946},[879,2341,2342,2345,2347,2349,2351,2353,2356,2358],{"class":881,"line":989},[879,2343,2344],{"class":982},"    s3_bucket_prefix",[879,2346,825],{"class":946},[879,2348,1058],{"class":970},[879,2350,1031],{"class":946},[879,2352,998],{"class":946},[879,2354,2355],{"class":1001},"data\u002Finstruments\u002Flabchip",[879,2357,1244],{"class":946},[879,2359,1276],{"class":946},[879,2361,2362,2364,2366],{"class":881,"line":1008},[879,2363,1318],{"class":946},[879,2365,1321],{"class":946},[879,2367,2368],{"class":946}," None:\n",[879,2370,2371,2374,2376,2379],{"class":881,"line":1013},[879,2372,2373],{"class":889},"    logger ",[879,2375,995],{"class":946},[879,2377,2378],{"class":1016}," get_run_logger",[879,2380,1791],{"class":946},[879,2382,2383],{"class":881,"line":1050},[879,2384,916],{"emptyLinePlaceholder":915},[879,2386,2387],{"class":881,"line":1065},[879,2388,2389],{"class":1061},"    # Load S3 credentials from secrets\n",[879,2391,2392,2394,2396,2398,2400,2402,2404],{"class":881,"line":1078},[879,2393,1565],{"class":885},[879,2395,1568],{"class":885},[879,2397,1571],{"class":1016},[879,2399,1574],{"class":946},[879,2401,1577],{"class":885},[879,2403,1580],{"class":889},[879,2405,1326],{"class":946},[879,2407,2408,2411,2413,2415,2417,2419,2422,2424,2426,2429,2431],{"class":881,"line":1089},[879,2409,2410],{"class":889},"        secrets ",[879,2412,995],{"class":946},[879,2414,1592],{"class":885},[879,2416,1580],{"class":889},[879,2418,791],{"class":946},[879,2420,2421],{"class":1016},"get",[879,2423,974],{"class":946},[879,2425,1244],{"class":946},[879,2427,2428],{"class":1001},"\u002Fsecrets",[879,2430,1244],{"class":946},[879,2432,1252],{"class":946},[879,2434,2435,2438,2440,2443,2445,2448,2450,2453,2455,2458,2460,2463,2465,2467,2469,2471,2473,2476,2478,2480,2482],{"class":881,"line":1099},[879,2436,2437],{"class":889},"    datalake ",[879,2439,995],{"class":946},[879,2441,2442],{"class":1016}," next",[879,2444,974],{"class":946},[879,2446,2447],{"class":1016},"s ",[879,2449,1975],{"class":885},[879,2451,2452],{"class":1016}," s ",[879,2454,1827],{"class":885},[879,2456,2457],{"class":1016}," secrets ",[879,2459,1986],{"class":885},[879,2461,2462],{"class":1016}," s",[879,2464,1110],{"class":946},[879,2466,1244],{"class":946},[879,2468,1239],{"class":1001},[879,2470,1244],{"class":946},[879,2472,1121],{"class":946},[879,2474,2475],{"class":946}," ==",[879,2477,998],{"class":946},[879,2479,781],{"class":1001},[879,2481,1244],{"class":946},[879,2483,1252],{"class":946},[879,2485,2486,2489,2491,2494,2496,2498,2501,2503],{"class":881,"line":1135},[879,2487,2488],{"class":889},"    p ",[879,2490,995],{"class":946},[879,2492,2493],{"class":889}," datalake",[879,2495,1110],{"class":946},[879,2497,1244],{"class":946},[879,2499,2500],{"class":1001},"parameters",[879,2502,1244],{"class":946},[879,2504,2000],{"class":946},[879,2506,2507,2510,2512],{"class":881,"line":1157},[879,2508,2509],{"class":889},"    s3_base ",[879,2511,995],{"class":946},[879,2513,2514],{"class":946}," (\n",[879,2516,2517,2520,2523,2527,2529,2531,2534,2537,2539,2541,2544,2546,2548,2550,2552,2554,2557,2559,2561,2563],{"class":881,"line":1163},[879,2518,2519],{"class":966},"        f",[879,2521,2522],{"class":1001},"\"s3s:\u002F\u002F",[879,2524,2526],{"class":2525},"sbssI","{",[879,2528,756],{"class":889},[879,2530,1110],{"class":946},[879,2532,2533],{"class":946},"'",[879,2535,2536],{"class":1001},"minio_root_user",[879,2538,2533],{"class":946},[879,2540,1121],{"class":946},[879,2542,2543],{"class":2525},"}",[879,2545,825],{"class":1001},[879,2547,2526],{"class":2525},[879,2549,756],{"class":889},[879,2551,1110],{"class":946},[879,2553,2533],{"class":946},[879,2555,2556],{"class":1001},"minio_root_password",[879,2558,2533],{"class":946},[879,2560,1121],{"class":946},[879,2562,2543],{"class":2525},[879,2564,1005],{"class":1001},[879,2566,2567,2569,2572,2574,2576,2578,2580,2583,2585,2588,2590,2593,2595,2598,2601,2603,2605,2608,2610,2612,2615,2617,2619],{"class":881,"line":1424},[879,2568,2519],{"class":966},[879,2570,2571],{"class":1001},"\"@",[879,2573,2526],{"class":2525},[879,2575,756],{"class":889},[879,2577,1110],{"class":946},[879,2579,2533],{"class":946},[879,2581,2582],{"class":1001},"aws_client_parameters",[879,2584,2533],{"class":946},[879,2586,2587],{"class":946},"][",[879,2589,2533],{"class":946},[879,2591,2592],{"class":1001},"endpoint_url",[879,2594,2533],{"class":946},[879,2596,2597],{"class":946},"].",[879,2599,2600],{"class":1016},"replace",[879,2602,974],{"class":946},[879,2604,2533],{"class":946},[879,2606,2607],{"class":1001},"http:\u002F\u002F",[879,2609,2533],{"class":946},[879,2611,947],{"class":946},[879,2613,2614],{"class":946}," ''",[879,2616,1318],{"class":946},[879,2618,2543],{"class":2525},[879,2620,1005],{"class":1001},[879,2622,2623,2625,2628,2630,2632,2634,2636,2638,2640,2642,2644,2647,2649,2651,2653,2656,2658,2661,2663],{"class":881,"line":1444},[879,2624,2519],{"class":966},[879,2626,2627],{"class":1001},"\"\u002F",[879,2629,2526],{"class":2525},[879,2631,756],{"class":889},[879,2633,1110],{"class":946},[879,2635,2533],{"class":946},[879,2637,2582],{"class":1001},[879,2639,2533],{"class":946},[879,2641,2587],{"class":946},[879,2643,2533],{"class":946},[879,2645,2646],{"class":1001},"bucket_name",[879,2648,2533],{"class":946},[879,2650,1121],{"class":946},[879,2652,2543],{"class":2525},[879,2654,2655],{"class":1001},"\u002F",[879,2657,2526],{"class":2525},[879,2659,2660],{"class":889},"s3_bucket_prefix",[879,2662,2543],{"class":2525},[879,2664,1005],{"class":1001},[879,2666,2667],{"class":881,"line":1961},[879,2668,1160],{"class":946},[879,2670,2671],{"class":881,"line":2003},[879,2672,916],{"emptyLinePlaceholder":915},[879,2674,2675,2678,2680,2683],{"class":881,"line":2011},[879,2676,2677],{"class":889},"    warehouse ",[879,2679,995],{"class":946},[879,2681,2682],{"class":1016}," get_warehouse_client",[879,2684,1791],{"class":946},[879,2686,2687,2689,2691,2693,2696],{"class":881,"line":2029},[879,2688,1427],{"class":885},[879,2690,1430],{"class":889},[879,2692,791],{"class":946},[879,2694,2695],{"class":1016},"ensure_tables_exist",[879,2697,1791],{"class":946},[879,2699,2700],{"class":881,"line":2051},[879,2701,916],{"emptyLinePlaceholder":915},[879,2703,2705],{"class":881,"line":2704},27,[879,2706,2707],{"class":1061},"    # ── Stage 1: Reconcile files missed while offline ──────────────────────\n",[879,2709,2711,2714,2716,2719,2721,2723,2726,2728],{"class":881,"line":2710},28,[879,2712,2713],{"class":889},"    logger",[879,2715,791],{"class":946},[879,2717,2718],{"class":1016},"info",[879,2720,974],{"class":946},[879,2722,1244],{"class":946},[879,2724,2725],{"class":1001},"Stage 1: Reconciling missed files",[879,2727,1244],{"class":946},[879,2729,1252],{"class":946},[879,2731,2733,2736,2738,2740,2742],{"class":881,"line":2732},29,[879,2734,2735],{"class":889},"    reconciliation ",[879,2737,995],{"class":946},[879,2739,1592],{"class":885},[879,2741,1693],{"class":1016},[879,2743,1132],{"class":946},[879,2745,2747,2749,2751,2753],{"class":881,"line":2746},30,[879,2748,1365],{"class":982},[879,2750,995],{"class":946},[879,2752,1370],{"class":1016},[879,2754,1276],{"class":946},[879,2756,2758,2760,2762,2764],{"class":881,"line":2757},31,[879,2759,1377],{"class":982},[879,2761,995],{"class":946},[879,2763,1382],{"class":1016},[879,2765,1276],{"class":946},[879,2767,2769,2772,2774,2777],{"class":881,"line":2768},32,[879,2770,2771],{"class":982},"        warehouse",[879,2773,995],{"class":946},[879,2775,2776],{"class":1016},"warehouse",[879,2778,1276],{"class":946},[879,2780,2782],{"class":881,"line":2781},33,[879,2783,1160],{"class":946},[879,2785,2787,2789,2791,2793],{"class":881,"line":2786},34,[879,2788,2713],{"class":889},[879,2790,791],{"class":946},[879,2792,2718],{"class":1016},[879,2794,1132],{"class":946},[879,2796,2798,2800,2803,2805,2808,2810,2813,2815,2817,2819,2821,2824,2826],{"class":881,"line":2797},35,[879,2799,2519],{"class":966},[879,2801,2802],{"class":1001},"\"Found ",[879,2804,2526],{"class":2525},[879,2806,2807],{"class":1016},"len",[879,2809,974],{"class":946},[879,2811,2812],{"class":1016},"reconciliation",[879,2814,1110],{"class":946},[879,2816,2533],{"class":946},[879,2818,2017],{"class":1001},[879,2820,2533],{"class":946},[879,2822,2823],{"class":946},"])",[879,2825,2543],{"class":2525},[879,2827,2828],{"class":1001}," unregistered files \"\n",[879,2830,2832,2834,2837,2839,2841,2843,2845,2847,2849,2851,2853],{"class":881,"line":2831},36,[879,2833,2519],{"class":966},[879,2835,2836],{"class":1001},"\"out of ",[879,2838,2526],{"class":2525},[879,2840,2812],{"class":1016},[879,2842,1110],{"class":946},[879,2844,2533],{"class":946},[879,2846,2034],{"class":1001},[879,2848,2533],{"class":946},[879,2850,1121],{"class":946},[879,2852,2543],{"class":2525},[879,2854,2855],{"class":1001}," on disk\"\n",[879,2857,2859],{"class":881,"line":2858},37,[879,2860,1160],{"class":946},[879,2862,2864],{"class":881,"line":2863},38,[879,2865,916],{"emptyLinePlaceholder":915},[879,2867,2869],{"class":881,"line":2868},39,[879,2870,2871],{"class":1061},"    # ── Stage 2: Process the backlog ───────────────────────────────────────\n",[879,2873,2875,2878,2880,2882,2885,2887,2889,2891,2893],{"class":881,"line":2874},40,[879,2876,2877],{"class":885},"    for",[879,2879,2182],{"class":889},[879,2881,1827],{"class":885},[879,2883,2884],{"class":889}," reconciliation",[879,2886,1110],{"class":946},[879,2888,1244],{"class":946},[879,2890,2017],{"class":1001},[879,2892,1244],{"class":946},[879,2894,2895],{"class":946},"]:\n",[879,2897,2899,2902,2904,2906,2908,2911,2913,2915,2917,2919,2922,2925],{"class":881,"line":2898},41,[879,2900,2901],{"class":889},"        filename ",[879,2903,995],{"class":946},[879,2905,2206],{"class":889},[879,2907,791],{"class":946},[879,2909,2910],{"class":1016},"split",[879,2912,974],{"class":946},[879,2914,1244],{"class":946},[879,2916,2655],{"class":1001},[879,2918,1244],{"class":946},[879,2920,2921],{"class":946},")[-",[879,2923,2924],{"class":2525},"1",[879,2926,2000],{"class":946},[879,2928,2930,2933,2935,2938,2940,2942,2945,2947,2949,2951,2954,2956],{"class":881,"line":2929},42,[879,2931,2932],{"class":889},"        s3_url ",[879,2934,995],{"class":946},[879,2936,2937],{"class":966}," f",[879,2939,1244],{"class":1001},[879,2941,2526],{"class":2525},[879,2943,2944],{"class":889},"s3_base",[879,2946,2543],{"class":2525},[879,2948,2655],{"class":1001},[879,2950,2526],{"class":2525},[879,2952,2953],{"class":889},"filename",[879,2955,2543],{"class":2525},[879,2957,1005],{"class":1001},[879,2959,2961],{"class":881,"line":2960},43,[879,2962,916],{"emptyLinePlaceholder":915},[879,2964,2966,2968,2970],{"class":881,"line":2965},44,[879,2967,1610],{"class":885},[879,2969,1517],{"class":1016},[879,2971,1132],{"class":946},[879,2973,2975,2978,2980,2982],{"class":881,"line":2974},45,[879,2976,2977],{"class":982},"            connector_name",[879,2979,995],{"class":946},[879,2981,1370],{"class":1016},[879,2983,1276],{"class":946},[879,2985,2987,2990,2992,2994],{"class":881,"line":2986},46,[879,2988,2989],{"class":982},"            file_path",[879,2991,995],{"class":946},[879,2993,1184],{"class":1016},[879,2995,1276],{"class":946},[879,2997,2999,3002,3004,3006],{"class":881,"line":2998},47,[879,3000,3001],{"class":982},"            s3_url",[879,3003,995],{"class":946},[879,3005,1642],{"class":1016},[879,3007,1276],{"class":946},[879,3009,3011],{"class":881,"line":3010},48,[879,3012,3013],{"class":946},"        )\n",[879,3015,3017,3019,3021],{"class":881,"line":3016},49,[879,3018,1610],{"class":885},[879,3020,1263],{"class":1016},[879,3022,1132],{"class":946},[879,3024,3026,3028,3030,3032],{"class":881,"line":3025},50,[879,3027,2989],{"class":982},[879,3029,995],{"class":946},[879,3031,1184],{"class":1016},[879,3033,1276],{"class":946},[879,3035,3037,3040,3042,3045,3047,3049,3051,3053,3055,3057,3059,3061,3063],{"class":881,"line":3036},51,[879,3038,3039],{"class":982},"            s3_object_path",[879,3041,995],{"class":946},[879,3043,3044],{"class":966},"f",[879,3046,1244],{"class":1001},[879,3048,2526],{"class":2525},[879,3050,2660],{"class":1016},[879,3052,2543],{"class":2525},[879,3054,2655],{"class":1001},[879,3056,2526],{"class":2525},[879,3058,2953],{"class":1016},[879,3060,2543],{"class":2525},[879,3062,1244],{"class":1001},[879,3064,1276],{"class":946},[879,3066,3068,3070,3072,3074],{"class":881,"line":3067},52,[879,3069,2977],{"class":982},[879,3071,995],{"class":946},[879,3073,1370],{"class":1016},[879,3075,1276],{"class":946},[879,3077,3079,3082,3084,3086],{"class":881,"line":3078},53,[879,3080,3081],{"class":982},"            instrument_type",[879,3083,995],{"class":946},[879,3085,1382],{"class":1016},[879,3087,1276],{"class":946},[879,3089,3091,3094,3096,3098],{"class":881,"line":3090},54,[879,3092,3093],{"class":982},"            warehouse",[879,3095,995],{"class":946},[879,3097,2776],{"class":1016},[879,3099,1276],{"class":946},[879,3101,3103],{"class":881,"line":3102},55,[879,3104,3013],{"class":946},[879,3106,3108],{"class":881,"line":3107},56,[879,3109,916],{"emptyLinePlaceholder":915},[879,3111,3113],{"class":881,"line":3112},57,[879,3114,3115],{"class":1061},"    # ── Stage 3: Real-time monitoring ──────────────────────────────────────\n",[879,3117,3119,3121,3123,3125,3127,3129,3132,3134],{"class":881,"line":3118},58,[879,3120,2713],{"class":889},[879,3122,791],{"class":946},[879,3124,2718],{"class":1016},[879,3126,974],{"class":946},[879,3128,1244],{"class":946},[879,3130,3131],{"class":1001},"Stage 3: Watching for new files in real time",[879,3133,1244],{"class":946},[879,3135,1252],{"class":946},[879,3137,3139,3141,3143,3146,3148,3150,3152,3154],{"class":881,"line":3138},59,[879,3140,1565],{"class":885},[879,3142,1943],{"class":885},[879,3144,3145],{"class":889}," change ",[879,3147,1827],{"class":885},[879,3149,2096],{"class":1016},[879,3151,974],{"class":946},[879,3153,1370],{"class":1016},[879,3155,2107],{"class":946},[879,3157,3159,3162,3164,3167,3169,3171,3173,3175],{"class":881,"line":3158},60,[879,3160,3161],{"class":889},"        file_path ",[879,3163,995],{"class":946},[879,3165,3166],{"class":889}," change",[879,3168,1110],{"class":946},[879,3170,1244],{"class":946},[879,3172,1184],{"class":1001},[879,3174,1244],{"class":946},[879,3176,2000],{"class":946},[879,3178,3180,3182,3184,3186,3188,3190,3192,3194,3196,3198,3200,3202],{"class":881,"line":3179},61,[879,3181,2901],{"class":889},[879,3183,995],{"class":946},[879,3185,2206],{"class":889},[879,3187,791],{"class":946},[879,3189,2910],{"class":1016},[879,3191,974],{"class":946},[879,3193,1244],{"class":946},[879,3195,2655],{"class":1001},[879,3197,1244],{"class":946},[879,3199,2921],{"class":946},[879,3201,2924],{"class":2525},[879,3203,2000],{"class":946},[879,3205,3207,3209,3211,3213,3215,3217,3219,3221,3223,3225,3227,3229],{"class":881,"line":3206},62,[879,3208,2932],{"class":889},[879,3210,995],{"class":946},[879,3212,2937],{"class":966},[879,3214,1244],{"class":1001},[879,3216,2526],{"class":2525},[879,3218,2944],{"class":889},[879,3220,2543],{"class":2525},[879,3222,2655],{"class":1001},[879,3224,2526],{"class":2525},[879,3226,2953],{"class":889},[879,3228,2543],{"class":2525},[879,3230,1005],{"class":1001},[879,3232,3234],{"class":881,"line":3233},63,[879,3235,916],{"emptyLinePlaceholder":915},[879,3237,3239,3242,3244,3246,3248,3250,3253,3255,3257,3259,3261],{"class":881,"line":3238},64,[879,3240,3241],{"class":889},"        logger",[879,3243,791],{"class":946},[879,3245,2718],{"class":1016},[879,3247,974],{"class":946},[879,3249,3044],{"class":966},[879,3251,3252],{"class":1001},"\"New file detected: ",[879,3254,2526],{"class":2525},[879,3256,1184],{"class":1016},[879,3258,2543],{"class":2525},[879,3260,1244],{"class":1001},[879,3262,1252],{"class":946},[879,3264,3266,3268,3270],{"class":881,"line":3265},65,[879,3267,1610],{"class":885},[879,3269,1517],{"class":1016},[879,3271,1132],{"class":946},[879,3273,3275,3277,3279,3281],{"class":881,"line":3274},66,[879,3276,2977],{"class":982},[879,3278,995],{"class":946},[879,3280,1370],{"class":1016},[879,3282,1276],{"class":946},[879,3284,3286,3288,3290,3292],{"class":881,"line":3285},67,[879,3287,2989],{"class":982},[879,3289,995],{"class":946},[879,3291,1184],{"class":1016},[879,3293,1276],{"class":946},[879,3295,3297,3299,3301,3303],{"class":881,"line":3296},68,[879,3298,3001],{"class":982},[879,3300,995],{"class":946},[879,3302,1642],{"class":1016},[879,3304,1276],{"class":946},[879,3306,3308],{"class":881,"line":3307},69,[879,3309,3013],{"class":946},[879,3311,3313,3315,3317],{"class":881,"line":3312},70,[879,3314,1610],{"class":885},[879,3316,1263],{"class":1016},[879,3318,1132],{"class":946},[879,3320,3322,3324,3326,3328],{"class":881,"line":3321},71,[879,3323,2989],{"class":982},[879,3325,995],{"class":946},[879,3327,1184],{"class":1016},[879,3329,1276],{"class":946},[879,3331,3333,3335,3337,3339,3341,3343,3345,3347,3349,3351,3353,3355,3357],{"class":881,"line":3332},72,[879,3334,3039],{"class":982},[879,3336,995],{"class":946},[879,3338,3044],{"class":966},[879,3340,1244],{"class":1001},[879,3342,2526],{"class":2525},[879,3344,2660],{"class":1016},[879,3346,2543],{"class":2525},[879,3348,2655],{"class":1001},[879,3350,2526],{"class":2525},[879,3352,2953],{"class":1016},[879,3354,2543],{"class":2525},[879,3356,1244],{"class":1001},[879,3358,1276],{"class":946},[879,3360,3362,3364,3366,3368],{"class":881,"line":3361},73,[879,3363,2977],{"class":982},[879,3365,995],{"class":946},[879,3367,1370],{"class":1016},[879,3369,1276],{"class":946},[879,3371,3373,3375,3377,3379],{"class":881,"line":3372},74,[879,3374,3081],{"class":982},[879,3376,995],{"class":946},[879,3378,1382],{"class":1016},[879,3380,1276],{"class":946},[879,3382,3384,3386,3388,3390],{"class":881,"line":3383},75,[879,3385,3093],{"class":982},[879,3387,995],{"class":946},[879,3389,2776],{"class":1016},[879,3391,1276],{"class":946},[879,3393,3395],{"class":881,"line":3394},76,[879,3396,3013],{"class":946},[815,3398,3400],{"id":3399},"run-the-flow","Run the flow",[870,3402,3406],{"className":3403,"code":3404,"language":3405,"meta":875,"style":875},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","uv run python -c \"\nimport asyncio\nfrom my_etl.flows.labchip import labchip_etl_flow\nasyncio.run(labchip_etl_flow())\n\"\n","bash",[773,3407,3408,3425,3430,3435,3440],{"__ignoreMap":875},[879,3409,3410,3413,3416,3419,3422],{"class":881,"line":882},[879,3411,3412],{"class":970},"uv",[879,3414,3415],{"class":1001}," run",[879,3417,3418],{"class":1001}," python",[879,3420,3421],{"class":1001}," -c",[879,3423,3424],{"class":946}," \"\n",[879,3426,3427],{"class":881,"line":899},[879,3428,3429],{"class":1001},"import asyncio\n",[879,3431,3432],{"class":881,"line":912},[879,3433,3434],{"class":1001},"from my_etl.flows.labchip import labchip_etl_flow\n",[879,3436,3437],{"class":881,"line":919},[879,3438,3439],{"class":1001},"asyncio.run(labchip_etl_flow())\n",[879,3441,3442],{"class":881,"line":933},[879,3443,1005],{"class":946},[815,3445,3447],{"id":3446},"deploy-with-prefect","Deploy with Prefect",[756,3449,3450,3451,3454],{},"Deploy the flow as a Prefect deployment so it restarts automatically on failure and appears in the Prefect UI for monitoring. See the ",[788,3452,499],{"href":3453},"\u002Fautomate\u002Fguides\u002Fdeploy-a-workflow\u002F"," guide for the manifest format and deployment commands.",[815,3456,3458],{"id":3457},"verify-in-the-warehouse","Verify in the Warehouse",[756,3460,3461,3462,3464],{},"After running the flow, query the ",[773,3463,1002],{}," table to confirm records were written:",[870,3466,3470],{"className":3467,"code":3468,"language":3469,"meta":875,"style":875},"language-sql shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","SELECT file_path, instrument_type, updated_at\nFROM warehouse.source_files\nWHERE instrument_type = 'labchip'\nORDER BY updated_at DESC\nLIMIT 20;\n","sql",[773,3471,3472,3477,3482,3487,3492],{"__ignoreMap":875},[879,3473,3474],{"class":881,"line":882},[879,3475,3476],{},"SELECT file_path, instrument_type, updated_at\n",[879,3478,3479],{"class":881,"line":899},[879,3480,3481],{},"FROM warehouse.source_files\n",[879,3483,3484],{"class":881,"line":912},[879,3485,3486],{},"WHERE instrument_type = 'labchip'\n",[879,3488,3489],{"class":881,"line":919},[879,3490,3491],{},"ORDER BY updated_at DESC\n",[879,3493,3494],{"class":881,"line":933},[879,3495,3496],{},"LIMIT 20;\n",[848,3498,3500],{"icon":3499},"i-heroicons-magnifying-glass",[756,3501,3502,3503,3507],{},"See ",[788,3504,3506],{"href":3505},"\u002Fobserve\u002Fconcepts\u002Fdata-sources\u002F","Data Sources"," for instructions on connecting DBeaver to the warehouse so you can browse tables without writing SQL.",[815,3509,3511],{"id":3510},"next-steps","Next steps",[766,3513,3514,3520,3526],{},[769,3515,3516,3519],{},[788,3517,544],{"href":3518},"\u002Fobserve\u002Fconcepts\u002Fstorage\u002F"," — understand how S3 URLs are constructed and how to browse raw files",[769,3521,3522,3525],{},[788,3523,560],{"href":3524},"\u002Fobserve\u002Fguides\u002Ffilesystem-connector\u002F"," — full reference for all SDK connector operations",[769,3527,3528,3530],{},[788,3529,3506],{"href":3505}," — query the Warehouse from DBeaver or any PostgreSQL client",[3532,3533,3534],"style",{},"html pre.shiki code .s7zQu, html code.shiki .s7zQu{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#89DDFF;--shiki-default-font-style:italic;--shiki-dark:#89DDFF;--shiki-dark-font-style:italic}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .spNyl, html code.shiki .spNyl{--shiki-light:#9C3EDA;--shiki-default:#C792EA;--shiki-dark:#C792EA}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sHdIc, html code.shiki .sHdIc{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#EEFFFF;--shiki-default-font-style:italic;--shiki-dark:#BABED8;--shiki-dark-font-style:italic}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .s2Zo4, html code.shiki .s2Zo4{--shiki-light:#6182B8;--shiki-default:#82AAFF;--shiki-dark:#82AAFF}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}",{"title":875,"searchDepth":882,"depth":899,"links":3536},[3537,3538,3539,3540,3541,3542,3543,3544,3545,3546,3547],{"id":817,"depth":899,"text":818},{"id":856,"depth":899,"text":857},{"id":1173,"depth":899,"text":1174},{"id":1453,"depth":899,"text":1454},{"id":1654,"depth":899,"text":1655},{"id":2057,"depth":899,"text":2058},{"id":2211,"depth":899,"text":2212},{"id":3399,"depth":899,"text":3400},{"id":3446,"depth":899,"text":3447},{"id":3457,"depth":899,"text":3458},{"id":3510,"depth":899,"text":3511},"Extract, transform, and load data between lab instruments and external systems using UniteLabs.","md",null,{},{"title":564,"description":3548},"4TqBJ8KfEB-4c3_oiVUOlhmGxqo8AUZmf7EGbuT9gNE",[3555,3557],{"title":560,"path":561,"stem":562,"description":3556},"A practical example of using the file system connector to read and write data in workflows.",{"title":573,"path":574,"stem":575,"description":3558},"The reference documentation.",1783607806333]