Skip to content

cdm_data_loaders.pipelines.xmltopyarrow.pipeline

XML to pyarrow pipeline for the KBase CTS.

run_xmltopyarrow_pipeline 

run_xmltopyarrow_pipeline(
    settings: XmlToPyArrowSettings,
) -> LoadInfo | None

Run the XML to PyArrow pipeline on matching files in settings.input_dir.

Parameters:

Name Type Description Default
settings XmlToPyArrowSettings

pipeline configuration

 required

Returns:

Type Description
LoadInfo | None

DLT pipeline LoadInfo or None
Source code in src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def run_xmltopyarrow_pipeline(settings: XmlToPyArrowSettings) -> LoadInfo | None:
    """Run the XML to PyArrow pipeline on matching files in settings.input_dir.

    :param settings: pipeline configuration
    :type  settings: XmlToPyArrowSettings
    :return: DLT pipeline LoadInfo or None
    :rtype : LoadInfo | None
    """
    xmltopyarrow_resource = build_per_file_filesystem_resource(
        settings,
        process_xml_file_to_pyarrow,
        name="xmltopyarrow_reader",
        parallelized=False,
    )
    pipeline_name = str(settings.model_config.get("cli_prog_name") or PIPELINE_NAME)

    return run_pipeline(
        settings=settings,
        resource=xmltopyarrow_resource,
        **build_file_ingest_kwargs(settings, pipeline_name),
    )