ɯei ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZmZmZmZddlmZddlZddlmcmcmcmcmZddlmZmZmZmZm Z ddl!m"Z"ddl#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+dd l,m-Z-dd l.m/Z/dd l0m1Z1ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ddlm?Z?m@Z@mAZAmBZBddlCmDZDddlEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYddlZm[Z[m\Z\m]Z]ddl^m_Z_ddl`maZambZbddlcmdZdmeZemfZfmgZgmhZhddlimjZjddlkmlZlddlmmnZnmoZompZpmqZqejdkrddl msZsnddltmsZseeuZvdZwddd d!d"d#d$Zxd%d&d'd'd(d%d)d*d+d,d-d. Zyd/Zzd0e{d1e{fd2Z|d3e}d1e}fd4Z~Gd5d6Zy)7N) defaultdict) getLogger)AnyDictListLiteralOptionalTupleUnionCallable)datetime)convert_value_to_sql_optioncreate_file_format_statement$drop_file_format_if_exists_statementinfer_schema_statementquote_name_without_upper_casing) Attribute) ReadFileNode)Alias)build_expr_from_python_valbuild_proto_from_struct_typebuild_table_namewith_src_position)JDBC)DataSourcePartitioner) Connection)STATEMENT_PARAMS_DATA_SOURCEDATA_SOURCE_DBAPI_SIGNATURE"create_data_source_table_and_stagelocal_ingestion!STATEMENT_PARAMS_DATA_SOURCE_JDBCDATA_SOURCE_JDBC_SIGNATURE get_jdbc_dbms)SnowparkClientExceptionMessages)set_api_call_source) ColumnOrNameconvert_sf_to_sp_typeconvert_sp_to_sf_typemost_permissive_type)get_types_from_type_hints)SNOWURL_PREFIX STAGE_PREFIXXML_ROW_TAG_STRINGXML_ROW_DATA_COLUMN_NAMEXML_READER_FILE_PATHXML_READER_API_SIGNATUREXML_READER_SQL_COMMENTINFER_SCHEMA_FORMAT_TYPESSNOWFLAKE_PATH_PREFIXESTempObjectTypeget_aliased_option_nameget_copy_into_table_optionsget_stage_partsis_in_stored_procedure&parse_positional_args_to_list_variadicprivate_preview publicapirandom_name_for_temp_objectwarning experimental)METADATA_COLUMN_TYPESColumn_to_col_if_str) DataFrame)SnowparkSessionException SnowparkDataframeReaderException)sql_exprcolconcatlitto_file)MockServerConnection)Table) StructTypeTimestampTimeZone VariantType StructField) )Iterable)JSONtime_travel_mode statementoffset timestamptimestamp_typestream)TIME_TRAVEL_MODE STATEMENTOFFSET TIMESTAMPTIMESTAMP_TYPESTREAMFIELD_DELIMITER PARSE_HEADERPATTERN INFER_SCHEMARECORD_DELIMITERFIELD_OPTIONALLY_ENCLOSED_BYNULL_IF DATE_FORMATTIMESTAMP_FORMAT) DELIMITERHEADERPATHGLOBFILTERFILENAMEPATTERN INFERSCHEMASEPLINESEPQUOTE NULLVALUE DATEFORMATTIMESTAMPFORMATrPpathreturncx|jdtfdtDstd|d|S)Nz"'c3@K|]}j|ywN) startswith).0prefix stripped_paths e/var/www/html/glpi_dashboard/venv/lib/python3.12/site-packages/snowflake/snowpark/dataframe_reader.py z'_validate_stage_path..sVF}''/Vs'zc' is an invalid Snowflake stage location. DataFrameReader can only read files from stage locations.)stripanyr3 ValueError)rtr|s @r}_validate_stage_pathrsAJJu%M V>UV Vvx y   Koptionsc&i}t}d|vrRd|vr!|djdk(r tdd|vr tdd|d<|d|d <|jdtj D]\}}||vs ||vs||||<|S) aB Extract time travel options from DataFrameReader options. Special handling for 'AS-OF-TIMESTAMP' (PySpark compatibility): - Automatically sets time_travel_mode to 'at' - Cannot be used with time_travel_mode='before' (raises error) - Cannot be mixed with regular 'timestamp' option (raises error) zAS-OF-TIMESTAMPrZbeforezBCannot use 'as-of-timestamp' option with time_travel_mode='before'r]z:Cannot use both 'as-of-timestamp' and 'timestamp' options.atrTrW)setlowerradd_TIME_TRAVEL_OPTIONS_PARAMS_MAPitems)rresult excluded_keys option_key param_names r}!_extract_time_travel_from_optionsrsFEMG# ' )*+113x?T  ' !L &*!"%&78{+&"A"G"G"I5 J  Z}%D!(!4F: 5 Mrc)eZdZdZe dSdddeddfdZed Zde e e e e ffd Z e dSddddddd d ee ee fded eeddee deedeee efdeee efdee defdZedSdededdfdZedddeededdfdZedee fdZej8de ddfdZe dSdeddeddfdZedTd ee dedefd!ZedSd e dedefd"Z edSd e dedefd#Z!edSd e dedefd$Z"edSd e dedefd%Z#edSd e dedefd&Z$edSd e dedefd'Z%edSd(e de&deddfd)Z'e dTd*ee(deddfd+Z)d e de de e e e e*ffd,Z+d-ede e e e ffd.Z,d e de defd/Z-e.d01edddddddd2d2ddddd3 d4e d5e/d6ee/d7ee d8ee d9ee d:eee efd;eee efdeed?eee efd@ee e dAeee e e fdedef dBZ0e.dC1edddddddd2dDdddddEdFdddGdHe1dId7ee d8ee d9ee d:eee efd;eee efdeed?eee efd@ee e dAeee e e fd5ee/dKedLedMee/dedef&dNZ2ee3dO1dSd e dedefdPZ4dSdQe dedefdRZ5y)UDataFrameReaderaW=Provides methods to load data in various supported formats from a Snowflake stage to a :class:`DataFrame`. The paths provided to the DataFrameReader must refer to Snowflake stages. To use this object: 1. Access an instance of a :class:`DataFrameReader` by using the :attr:`Session.read` property. 2. Specify any `format-specific options `_ and `copy options `_ by calling the :func:`option` or :func:`options` method. These methods return a DataFrameReader that is configured with these options. (Note that although specifying copy options can make error handling more robust during the reading process, it may have an effect on performance.) 3. Specify the schema of the data that you plan to load by constructing a :class:`types.StructType` object and passing it to the :func:`schema` method if the file format is CSV. Other file formats such as JSON, XML, Parquet, ORC, and AVRO don't accept a schema. This method returns a :class:`DataFrameReader` that is configured to read data that uses the specified schema. Currently, inferring schema is also supported for CSV and JSON formats as a preview feature open to all accounts. 4. Specify the format of the data by calling the method named after the format (e.g. :func:`csv`, :func:`json`, etc.). These methods return a :class:`DataFrame` that is configured to load data in the specified format. 5. Call a :class:`DataFrame` method that performs an action (e.g. :func:`DataFrame.collect`) to load the data from the file. The following examples demonstrate how to use a DataFrameReader. >>> # Create a temp stage to run the example code. >>> _ = session.sql("CREATE or REPLACE temp STAGE mystage").collect() Example 1: Loading the first two columns of a CSV file and skipping the first header line: >>> from snowflake.snowpark.types import StructType, StructField, IntegerType, StringType, FloatType >>> _ = session.file.put("tests/resources/testCSV.csv", "@mystage", auto_compress=False) >>> # Define the schema for the data in the CSV file. >>> user_schema = StructType([StructField("a", IntegerType()), StructField("b", StringType()), StructField("c", FloatType())]) >>> # Create a DataFrame that is configured to load data from the CSV file. >>> df = session.read.options({"field_delimiter": ",", "skip_header": 1}).schema(user_schema).csv("@mystage/testCSV.csv") >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.collect() [Row(A=2, B='two', C=2.2)] Example 2: Loading a gzip compressed json file: >>> _ = session.file.put("tests/resources/testJson.json", "@mystage", auto_compress=True) >>> # Create a DataFrame that is configured to load data from the gzipped JSON file. >>> json_df = session.read.option("compression", "gzip").json("@mystage/testJson.json.gz") >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> json_df.show() ----------------------- |"$1" | ----------------------- |{ | | "color": "Red", | | "fruit": "Apple", | | "size": "Large" | |} | ----------------------- In addition, if you want to load only a subset of files from the stage, you can use the `pattern `_ option to specify a regular expression that matches the files that you want to load. Example 3: Loading only the CSV files from a stage location: >>> from snowflake.snowpark.types import StructType, StructField, IntegerType, StringType >>> from snowflake.snowpark.functions import col >>> _ = session.file.put("tests/resources/*.csv", "@mystage", auto_compress=False) >>> # Define the schema for the data in the CSV files. >>> user_schema = StructType([StructField("a", IntegerType()), StructField("b", StringType()), StructField("c", FloatType())]) >>> # Create a DataFrame that is configured to load data from the CSV files in the stage. >>> csv_df = session.read.option("pattern", ".*V[.]csv").schema(user_schema).csv("@mystage").sort(col("a")) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> csv_df.collect() [Row(A=1, B='one', C=1.2), Row(A=2, B='two', C=2.2), Row(A=3, B='three', C=3.3), Row(A=4, B='four', C=4.4)] To load Parquet, ORC and AVRO files, no schema is accepted because the schema will be automatically inferred. Inferring the schema can be disabled by setting option "infer_schema" to ``False``. Then you can use ``$1`` to access the column data as an OBJECT. Example 4: Loading a Parquet file with inferring the schema. >>> from snowflake.snowpark.functions import col >>> _ = session.file.put("tests/resources/test.parquet", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.parquet("@mystage/test.parquet").where(col('"num"') == 2) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.collect() [Row(str='str2', num=2)] Example 5: Loading an ORC file and infer the schema: >>> from snowflake.snowpark.functions import col >>> _ = session.file.put("tests/resources/test.orc", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.orc("@mystage/test.orc").where(col('"num"') == 2) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.collect() [Row(str='str2', num=2)] Example 6: Loading an AVRO file and infer the schema: >>> from snowflake.snowpark.functions import col >>> _ = session.file.put("tests/resources/test.avro", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.avro("@mystage/test.avro").where(col('"num"') == 2) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.collect() [Row(str='str2', num=2)] Example 7: Loading a Parquet file without inferring the schema: >>> from snowflake.snowpark.functions import col >>> _ = session.file.put("tests/resources/test.parquet", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.option("infer_schema", False).parquet("@mystage/test.parquet").where(col('$1')["num"] == 2) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.show() ------------------- |"$1" | ------------------- |{ | | "num": 2, | | "str": "str2" | |} | ------------------- Loading JSON and XML files doesn't support schema either. You also need to use ``$1`` to access the column data as an OBJECT. Example 8: Loading a JSON file: >>> from snowflake.snowpark.functions import col, lit >>> _ = session.file.put("tests/resources/testJson.json", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.json("@mystage/testJson.json").where(col("$1")["fruit"] == lit("Apple")) >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.show() ----------------------- |"$1" | ----------------------- |{ | | "color": "Red", | | "fruit": "Apple", | | "size": "Large" | |} | |{ | | "color": "Red", | | "fruit": "Apple", | | "size": "Large" | |} | ----------------------- Example 9: Loading an XML file: >>> _ = session.file.put("tests/resources/test.xml", "@mystage", auto_compress=False) >>> # Create a DataFrame that uses a DataFrameReader to load data from a file in a stage. >>> df = session.read.xml("@mystage/test.xml") >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.show() --------------------- |"$1" | --------------------- | | | 1 | | str1 | | | | | | 2 | | str2 | | | --------------------- Example 10: Loading a CSV file with an already existing FILE_FORMAT: >>> from snowflake.snowpark.types import StructType, StructField, IntegerType, StringType >>> _ = session.sql("create file format if not exists csv_format type=csv skip_header=1 null_if='none';").collect() >>> _ = session.file.put("tests/resources/testCSVspecialFormat.csv", "@mystage", auto_compress=False) >>> # Define the schema for the data in the CSV files. >>> schema = StructType([StructField("ID", IntegerType()),StructField("USERNAME", StringType()),StructField("FIRSTNAME", StringType()),StructField("LASTNAME", StringType())]) >>> # Create a DataFrame that is configured to load data from the CSV files in the stage. >>> df = session.read.schema(schema).option("format_name", "csv_format").csv("@mystage/testCSVspecialFormat.csv") >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.collect() [Row(ID=0, USERNAME='admin', FIRSTNAME=None, LASTNAME=None), Row(ID=1, USERNAME='test_user', FIRSTNAME='test', LASTNAME='user')] Example 11: Querying metadata for staged files: >>> from snowflake.snowpark.column import METADATA_FILENAME, METADATA_FILE_ROW_NUMBER >>> df = session.read.with_metadata(METADATA_FILENAME, METADATA_FILE_ROW_NUMBER.as_("ROW NUMBER")).schema(user_schema).csv("@mystage/testCSV.csv") >>> # Load the data into the DataFrame and return an array of rows containing the results. >>> df.show() -------------------------------------------------------- |"METADATA$FILENAME" |"ROW NUMBER" |"A" |"B" |"C" | -------------------------------------------------------- |testCSV.csv |1 |1 |one |1.2 | |testCSV.csv |2 |2 |two |2.2 | -------------------------------------------------------- Example 12: Inferring schema for csv and json files (Preview Feature - Open): >>> # Read a csv file without a header >>> df = session.read.option("INFER_SCHEMA", True).csv("@mystage/testCSV.csv") >>> df.show() ---------------------- |"c1" |"c2" |"c3" | ---------------------- |1 |one |1.2 | |2 |two |2.2 | ---------------------- >>> # Read a csv file with header and parse the header >>> _ = session.file.put("tests/resources/testCSVheader.csv", "@mystage", auto_compress=False) >>> df = session.read.option("INFER_SCHEMA", True).option("PARSE_HEADER", True).csv("@mystage/testCSVheader.csv") >>> df.show() ---------------------------- |"id" |"name" |"rating" | ---------------------------- |1 |one |1.2 | |2 |two |2.2 | ---------------------------- >>> df = session.read.option("INFER_SCHEMA", True).json("@mystage/testJson.json") >>> df.show() ------------------------------ |"color" |"fruit" |"size" | ------------------------------ |Red |Apple |Large | |Red |Apple |Large | ------------------------------ Example 13: Reading an XML file with a row tag >>> # Each XML record is extracted as a separate row, >>> # and each field within that record becomes a separate column of type VARIANT >>> _ = session.file.put("tests/resources/nested.xml", "@mystage", auto_compress=False) >>> df = session.read.option("rowTag", "tag").xml("@mystage/nested.xml") >>> df.show() ----------------------- |"'test'" | ----------------------- |{ | | "num": "1", | | "obj": { | | "bool": "true", | | "str": "str2" | | }, | | "str": "str1" | |} | ----------------------- >>> # Query nested fields using dot notation >>> from snowflake.snowpark.functions import col >>> df.select( ... "'test'.num", "'test'.str", col("'test'.obj"), col("'test'.obj.bool") ... ).show() ------------------------------------------------------------------------------------------------------ |"""'TEST'"":""NUM""" |"""'TEST'"":""STR""" |"""'TEST'"":""OBJ""" |"""'TEST'"":""OBJ"".""BOOL""" | ------------------------------------------------------------------------------------------------------ |"1" |"str1" |{ |"true" | | | | "bool": "true", | | | | | "str": "str2" | | | | |} | | ------------------------------------------------------------------------------------------------------ Tsessionz"snowflake.snowpark.session.Session _emit_astruNc||_d|_i|_d|_d|_d|_d|_d|_d|_ddg|_ d|_ |r1tj}t|j||_ yy)Njdbcdbapi)_session _user_schema _cur_options _file_path _file_type_metadata_cols_infer_schema_transformations_infer_schema_target_columns_DataFrameReader__format_data_source_format_astprotoExprrdataframe_reader)selfrrreaders r}__init__zDataFrameReader.__init__s  26,.)-)-@D  *BF)'+ $*G#4  ZZ\F f55 6DI rc`|jtvr|jjddSy)NrcTF)rr2rgetrs r} _infer_schemazDataFrameReader._infer_schemas- ??7 7$$((> >rc |jrH|jDcgc]2}|jjj|ji4}}ng}g}dt dt fd} |jxsgDcgc]A}t|jjt||jC}}||fScc}wcc}w#t$r$tdtjd|wxYw)N unaliasedruct|trTt|jtr |jjj S|j jS|Srx) isinstancer@ _expressionrchildsql_namedname)rs r}_get_unaliased_namezMDataFrameReader._get_metadata_project_and_schema.._get_unaliased_namesO)V,i33U;$0066::: '')... rz1Metadata column name is not supported. Supported z, Got )rr _analyzeranalyzerr&strrrrr?upperKeyErrorrkeys)rrFmetadata_projectmetadata_schemar metadata_cols r} _get_metadata_project_and_schemaz0DataFrameReader._get_metadata_project_and_schemas    ..  ''//D    "  < C   %)$7$7$=2  !  '')..)*=l*K*Q*Q*STO 00; " CDYD^D^D`Caaghxgyz  s$7C*C>> table = session.read.table("my_table", time_travel_mode="at", offset=-3600) # doctest: +SKIP # Using options (recommended for chaining) >>> table = (session.read # doctest: +SKIP ... .option("time_travel_mode", "at") ... .option("offset", -3600) ... .table("my_table")) # PySpark-style as-of-timestamp (automatically sets mode to "at") >>> table = session.read.option("as-of-timestamp", "2023-01-01 12:00:00").table("my_table") # doctest: +SKIP # timestamp_type automatically set to "TZ" due to timezone info >>> import datetime, pytz # doctest: +SKIP >>> tz_aware = datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=pytz.UTC) # doctest: +SKIP >>> table1 = session.read.table("my_table", time_travel_mode="at", timestamp=tz_aware) # doctest: +SKIP # timestamp_type remains "NTZ" (user's explicit choice respected) >>> table2 = session.read.table("my_table", time_travel_mode="at", timestamp=tz_aware, timestamp_type="NTZ") # doctest: +SKIP # Mixing options and parameters (direct parameters completely override options) >>> table = (session.read # doctest: +SKIP ... .option("time_travel_mode", "before") # This will be IGNORED ... .option("offset", -60) # This will be IGNORED ... .table("my_table", time_travel_mode="at", offset=-3600)) # Only this is used NrrF)rr _ast_batchbindrexpr read_tablerCopyFromrrrTvaluerUrVrrWrrXrYrrtableuid_ast_id) rrrrTrUrVrWrXrYstmtasttime_travel_paramsrs r}rzDataFrameReader.table sLL .==++002D#DII$8$8$?C JJ   * SXXt ,+-=$$*$&/ #!#)  $*3==)D)+.~+>""(!#)   '$4& &"0 " "C4CTCT!U # ##DPEP=OP ) HHEM rschemac|r6|j*t||jjj||_|S)aDefine the schema for CSV files that you want to read. Args: schema: Schema configuration for the CSV file to be read. Returns: a :class:`DataFrameReader` instance with the specified schema configuration for the data to be read. )rrrrr)rrrs r}rzDataFrameReader.schemaws7 . (1K1K1R1R S" rr metadata_colsct|jjtr+|jjj dt |r|j yt|\}}||j jj_ |D]D}t|j jjjj|F|Dcgc]}t|dc}|_|Scc}w)a Define the metadata columns that need to be selected from stage files. Returns: a :class:`DataFrameReader` instance with metadata columns to read. See Also: https://docs.snowflake.com/en/user-guide/querying-metadata zDataFrameReader.with_metadata)external_feature_name raise_error)rr_connrJlog_not_supported_errorNotImplementedErrorrr9rmetadata_columnsvariadicrargsrrAr)rrr col_names is_variadicerFs r} with_metadatazDataFrameReader.with_metadatas dmm))+? @ MM   7 7&E/ 8  .%K& "I{DODII & & 7 7 @ *II..??DDHHJA %  3 ? @   sC8c|jSrx)rrs r}_formatzDataFrameReader._formats }}rrc|jj}gd}|j|j||vrt d|d|d||_y)N)csvjsonavroparquetorcxmlInvalid format 'z'. Supported formats are .)rrextendrrr)rr canon_formatallowed_formatss r}rzDataFrameReader._formatsf{{}**,   t778  ."5')B?BSSTU % rformat)rrrrrrrrc||_|r;|j/|j|jjj_|S)aSpecify the format of the file(s) to load. Args: format: The format of the file(s) to load. Supported formats are csv, json, avro, parquet, orc, and xml. Snowpark data source type is also supported, currently support dbapi and jdbc. Returns: a :class:`DataFrameReader` instance that is set up to load data from the specified file format in a Snowflake stage. )rrrrr)rrrs r}rzDataFrameReader.formats8" .6:llDII & & - - 3 rrtc |j td|jj}||jvr|td|d||jvr | t d|dk(rL|j d i|j jDcic]\}}|j|c}}S|dk(rL|jd i|j jDcic]\}}|j|c}}St||jd}|||d }|r|j|jjj}t|jj |} || _| j$j'|j|j(|_|Std |jd cc}}wcc}}w) aZSpecify the path of the file(s) to load. Args: path: The stage location of a file, or a stage location that has files. This parameter is required for all formats except dbapi. Returns: a :class:`DataFrame` that is set up to load data from the specified file(s) in a Snowflake stage. NzKPlease specify the format of the file(s) to load using the format() method.z.The 'path' parameter is not supported for the z1 format. Please omit this parameter when calling.zEDataFrameReader.load() missing 1 required positional argument: 'path'rrFrrz'.)rrrr TypeErrorrrrrgetattrrrrrrr read_loadrtrrrr) rrtr format_strkvloaderresrrs r}loadzDataFrameReader.loads << ] \\'') 11 1d6F@ L}~  T55 5$,W   4::U$:K:K:Q:Q:S T$!QA TU U  499T9J9J9P9P9RSA 1 ST Tt||T2  /CTYY2}}//446' (;(;TB ##DII."hh J+DLL><==!!USs G-G!ct|}||_d|_d\}}|js|jst j t|jjtr1|jjjddddddt |j|d\}}}}|pt|tr|tj!d |d t#d t%dg}dg}g}n)d|j&d<|jj)}|j+\}}d } |r||j,p|jj.j1} t3| j4j6| } || _| j:j=|j,|jj>rtA|j|jjBjE|jjBjGtI||j|j&||||||jjB|jjB| |} n>tA|jtI||j|j&|||||| |} || _%tM| d| S)a$Specify the path of the CSV file(s) to load. Args: path: The stage location of a CSV file, or a stage location that has CSV files. Returns: a :class:`DataFrame` that is set up to load data from the specified CSV file(s) in a Snowflake stage. CSV)NNz?Read option 'INFER_SCHEMA of value 'TRUE' for file format 'csv'zDataFrameReader.csvrrcTRUE)roption option_valuerinternal_feature_nameparameters_inforNz-Could not infer csv schema due to exception: zu. Using schema (C1, VariantType()) instead. Please use DataFrameReader.schema() to specify user schema for the file.z"C1"T)z$1C1F)schema_to_casttransformationsrranalyzerfrom_r ) _ast_stmtr)'rrrrrr$'DF_MUST_PROVIDE_SCHEMA_FOR_READING_FILErrrrJrr_infer_schema_for_file_formatFileNotFoundErrorloggerr=rrNr_to_attributesrrrrrrread_csvrtrrsql_simplifier_enabledrBrcreate_select_statementcreate_select_snowflake_planr_readerr%) rrtrrrr exceptionrrrrdfs r}rzDataFrameReader.csvs$D)*4'  %%5]]__$----/CD ##;;*k*?"'"0(.% !4< 224?  $i):;#OCI;O;; $FKM4@A"."$05D  n -&&557F,0,Q,Q,S)/ .==++002D#DII$6$6=CCH JJ   * == / /  ''??--11NN$  OO --"+9,;-=,; "&!8!8O "]]44@ #'B, OO%%#1$3%5$3 #B B 56 rcd|jvrd|jd<|j|d}|r|j|jjj }t |jj|}||_ |jj|j|j|_ |S)a(Specify the path of the JSON file(s) to load. Args: path: The stage location of a JSON file, or a stage location that has JSON files. Returns: a :class:`DataFrame` that is set up to load data from the specified JSON file(s) in a Snowflake stage. rcFrS)r_read_semi_structured_filerrrrrr read_jsonrtrrrrrrtrrrrs r}rzDataFrameReader.jsonos !2!2 205D  n -  , ,T6 : .==++002D#DII$7$7>CCH JJ   *BJ rcH|j|d}|r|j|jjj }t |j j|}||_|jj|j|j|_ |S)aSpecify the path of the AVRO file(s) to load. Args: path: The stage location of an AVRO file, or a stage location that has AVRO files. Note: When using :meth:`DataFrame.select`, quote the column names to select the desired columns. This is needed because converting from AVRO to `class`:`DataFrame` does not capitalize the column names from the original columns and a :meth:`DataFrame.select` without quote looks for capitalized column names. Returns: a :class:`DataFrame` that is set up to load data from the specified AVRO file(s) in a Snowflake stage. AVRO) rrrrrrr read_avrortrrrrrs r}rzDataFrameReader.avros  , ,T6 : .==++002D#DII$7$7>CCH JJ   *BJ rcH|j|d}|r|j|jjj }t |j j|}||_|jj|j|j|_ |S)aSpecify the path of the PARQUET file(s) to load. Args: path: The stage location of a PARQUET file, or a stage location that has PARQUET files. Note: When using :meth:`DataFrame.select`, quote the column names to select the desired columns. This is needed because converting from PARQUET to `class`:`DataFrame` does not capitalize the column names from the original columns and a :meth:`DataFrame.select` without quote looks for capitalized column names. Returns: a :class:`DataFrame` that is set up to load data from the specified PARQUET file(s) in a Snowflake stage. PARQUET) rrrrrrr read_parquetrtrrrrrs r}rzDataFrameReader.parquets" , ,T9 = .==++002D#DII$:$:DACCH JJ   *BJ rcH|j|d}|r|j|jjj }t |j j|}||_|jj|j|j|_ |S)aSpecify the path of the ORC file(s) to load. Args: path: The stage location of a ORC file, or a stage location that has ORC files. Note: When using :meth:`DataFrame.select`, quote the column names to select the desired columns. This is needed because converting from ORC to `class`:`DataFrame` does not capitalize the column names from the original columns and a :meth:`DataFrame.select` without quote looks for capitalized column names. Returns: a :class:`DataFrame` that is set up to load data from the specified ORC file(s) in a Snowflake stage. ORC) rrrrrrrread_orcrtrrrrrs r}rzDataFrameReader.orcs  , ,T5 9 .==++002D#DII$6$6=CCH JJ   *BJ rcH|j|d}|r|j|jjj }t |j j|}||_|jj|j|j|_ |S)aSpecify the path of the XML file(s) to load. Args: path: The stage location of an XML file, or a stage location that has XML files. Returns: a :class:`DataFrame` that is set up to load data from the specified XML file(s) in a Snowflake stage. Notes about reading XML files using a row tag: - We support reading XML by specifying the element tag that represents a single record using the ``rowTag`` option. See Example 13 in :class:`DataFrameReader`. - Each XML record is flattened into a single row, with each XML element or attribute mapped to a column. All columns are represented with the variant type to accommodate heterogeneous or nested data. Therefore, every column value has a size limit due to the variant type. - The column names are derived from the XML element names. It will always be wrapped by single quotes. - To parse the nested XML under a row tag, you can use dot notation ``.`` to query the nested fields in a DataFrame. See Example 13 in :class:`DataFrameReader`. - ``compression`` is not supported for XML files when ``rowTag`` is specified. - When ``rowTag`` is specified, the following options are supported for reading XML files via :meth:`option()` or :meth:`options()`: + ``mode``: Specifies the mode for dealing with corrupt XML records. The default value is ``PERMISSIVE``. The supported values are: - ``PERMISSIVE``: When it encounters a corrupt record, it sets all fields to null and includes a `columnNameOfCorruptRecord` column. - ``DROPMALFORMED``: Ignores the whole record that cannot be parsed correctly. - ``FAILFAST``: When it encounters a corrupt record, it raises an exception immediately. + ``columnNameOfCorruptRecord``: Specifies the name of the column that contains the corrupt record. The default value is '_corrupt_record'. + ``ignoreNamespace``: remove namespace prefixes from XML element names when constructing result column names. The default value is ``True``. Parsing uses recovery mode to tolerate malformed records (e.g., undefined namespace prefixes in attributes such as ``diffgr:id`` or ``msdata:rowOrder``). When this option is enabled, element name prefixes are stripped where resolvable; if a prefix isn't declared on the row tag element, it cannot be resolved and will be left intact (i.e. this setting is ignored for that element). For example, for the following XML data with a row tag ``abc:def``: ``` 0 ``` the result column name is ``abc:xyz`` where ``abc`` is not stripped. + ``attributePrefix``: The prefix to add to the attribute names. The default value is ``_``. + ``excludeAttributes``: Whether to exclude attributes from the XML element. The default value is ``False``. + ``valueTag``: The column name used for the value when there are attributes in an element that has no child elements. The default value is ``_VALUE``. + ``nullValue``: The value to treat as a null value. The default value is ``""``. + ``charset``: The character encoding of the XML file. The default value is ``utf-8``. + ``ignoreSurroundingWhitespace``: Whether or not whitespaces surrounding values should be skipped. The default value is ``False``. + ``rowValidationXSDPath``: The Snowflake stage path to the XSD file used for row validation. Rows that do not match the XSD schema will be considered corrupt records and handled according to the specified ``mode`` option. + ``cacheResult``: Whether to cache the result DataFrame of the XML reader to a temporary table after calling :meth:`xml`. When set to ``True`` (default), the result is cached and all subsequent operations on the DataFrame are performed on the cached data. This means the actual computation occurs before :meth:`DataFrame.collect` is called. When set to ``False``, the DataFrame is computed lazily and the actual computation occurs when :meth:`DataFrame.collect` is called. XML) rrrrrrrread_xmlrtrrrrrs r}rzDataFrameReader.xmls~T , ,T5 9 .==++002D#DII$6$6=CCH JJ   *BJ rkeyc,|rr|jf|jjjj}||_t |j t|tr t|n|t|t}||j|<|S)aSets the specified option in the DataFrameReader. Use this method to configure any `format-specific options `_, `copy options `_, and `time travel options `_. (Note that although specifying copy options can make error handling more robust during the reading process, it may have an effect on performance.) Time travel options can be used with ``table()`` method: - ``time_travel_mode``: Either "at" or "before" - ``statement``: Query ID for statement-based time travel - ``offset``: Seconds to go back in time (negative integer) - ``timestamp``: Specific timestamp for time travel - ``timestamp_type``: Type of timestamp interpretation ('NTZ', 'LTZ', or 'TZ'). - ``stream``: Stream name for time travel. Special PySpark compatibility option: - ``as-of-timestamp``: Automatically sets ``time_travel_mode`` to "at" and uses the provided timestamp. Cannot be used with ``time_travel_mode="before"``. Args: key: Name of the option (e.g. ``compression``, ``skip_header``, ``time_travel_mode``, etc.). value: Value of the option. ) rrrr_1r_2rrMrr5READER_OPTIONS_ALIAS_MAPr)rr)rrt aliased_keys r}rzDataFrameReader.option2sz: . **22668AAD &Ju6G$Hc%je .c3KL ).+& rconfigsc |r |r td||s td|}|jD]\}}|j||||S)aRSets multiple specified options in the DataFrameReader. This method is the same as the :meth:`option` except that you can set multiple options in one call. Args: configs: Dictionary of the names of options (e.g. ``compression``, ``skip_header``, etc.) and their corresponding values. z]Cannot set options with both a dictionary and keyword arguments. Please use one or the other.zNo options were providedr)rrr)rr0rkwargsrrs r}rzDataFrameReader.optionsZsd vo  ? !;<<GMMO 3DAq KK1 K 2 3 rc V t|j\}}|jjt t j }d}d|jv}|jjd|}|jjdi} | jdd} |jjdd} |} | } d}|jjddx}rd| vr{|jtset|}|jjjd|d |d }t|r t|\}}t|d \}}|jjjd |d }|Dcgc] }|ddk(s |c}dd}|rt!j"|n|g}|Dcgc]<}t%j&dt%j(|t$j*>}}g}|D]1}|d}|D]}|j-d|}|j/|3|| d<|j1|d|z} d }t7| || xsd} |rV|jjjt9|||d d |jj:d d t=|} |jjj|d } t| dk(rt?d|dg}!g}"g}#d}$| D]y}%|%d|%d|%d|%df\}&}'}(})tA|&}*|'jCd}+t|+},|,dk(r|'}-d}.d}/nH|+d}-tE|+djCdd}.tE|+djCdddd}/tG|-|.|/d|jjjH}0| r tK|0}0|!j/tM|*|0|(| r'|)jCd d}1d!|1d"tO|0d#}2n7|d$k(r|)}2n/|d%k(r"|)jQjd&r|)}2nd'|*d(|'}2|"j/|2|&f|#j/tS|2|tUjV|!|_,|#|_-|jXj\|_/|#D3cgc]}3|3j`jb}$}3 |(|jjj|d  |!|"|$dfScc}wcc}w#t2$r}t5dd|d|Yd}~d}~wwxYw#t2$rE}|r6t7| |d}|jjj|d } n|Yd}~d}~wwxYwcc}3w#t2$r:}ddd|fcYd}~|(|jjj|d SSd}~wwxYw#|(|jjj|d wwxYw))N FORMAT_NAMEINFER_SCHEMA_OPTIONSUSE_RELAXED_TYPESFTRY_CASTrbFILESzlist z pattern = dataT)return_full_stage_namezdescribe stage rSTAGE_LOCATIONrP^infer_schema_pattern_matchingz,Failed to infer schema for provided pattern z with error: )temp if_not_existuse_scoped_temp_objects is_generated)is_ddl_on_temp_objectz Given path: 'z!' could not be found or is empty.(,)nullable:z TRY_CAST(z AS )rr!GET_IGNORE_CASE$1:::)2r6rr$get_fully_qualified_name_if_possibler<r4 FILE_FORMATrpopryr+rr run_querylenr7rloadsrecompileescape IGNORECASEsubappend partition Exceptionr=rr_use_scoped_temp_objectsrrrsplitintr'max_string_sizer)rr(rrErL_from_attributesrrnamesrrr)4rrtrformat_type_options_temp_file_format_name$drop_tmp_file_format_if_exists_queryuse_temp_file_formatfile_format_nameinfer_schema_optionsuse_relaxed_typestry_cast infer_path fallback_pathshould_fallbackpatternescaped_patternmatchesstagefully_qualified_stagestage_descriptionxstage_locations_jsonstage_locationslocationregexesfilesmatchr{regexrinfer_schema_queryresults new_schemarrread_file_transformationsr column_nametyperI expressionrdata_type_parts parts_length data_type precisionscaledatatypeid identifierr.s4 r}rz-DataFrameReader._infer_schema_for_file_formatss "=T=N=N!OQ $ R R '(B(B C! ?C,,D4E4EE,,00@UV#00445KRP1445H%P$$((U;  " ))--i> >W >33OON3 :'BOmm))33v[(9:G7|0.t4HE1/>T0,)1)- (;(;(E(E)*?)@A))% $5,!@P8P,,,(0 #78#W$)8$ Qryy':&;#*F$ ] # ##--0(+!%)04 0V0V%)+/. 99IJ5 ----778JKFS7|q '#D6)JKJNHJO(, %* =:;A$!adAaD:P7 T8Z6{C#'**S/"?3 1$ $I !IE / 2I #OA$6$<$+DdJJU,&!7Fwi~^_]`a> ")?%'7*&#mm11;;+DDD%QsDcLt|jjtru|jjj r t dd|t vr<|jjjd|dddt|it|jr"|jd k7rtd |d t|}||_||_t!d t#g}d}d}d }|jr3|j%|j\}}}|}d |j&d<d}n'|j(r|j+||\}}}}|r|}|j-\} } |dk(rIt.|j&vr6t1rt3t4j6} d| dt8} |jj;| d j=d |jjj?t@| d ddtB| dtDjFjIt@} nt@} d}tKt@|ft4jL\}}tOtQtRt#dg}|jjTjW| |||ddgdd}nd}|jjXrt[|j|jj\j_|jj\jatc|||j&|||| | || |jj\|jj\d }n5t[|jtc|||j&|||| | || d }||_2|rEtg|th|j&jkddr|jm}d|_7|Stg|d|j|S)NzBCannot perform this operation because the session has been closed.1404) error_codezRead semi structured z filez*DataFrameReader._read_semi_structured_filerrrzRead z does not support user schemaz"$1"FrcTr'z create temp stage if not exists  r) compress_data overwriteskip_upload_on_content_match/ XMLReaderzsnowflake-snowpark-pythonzlxml<6) output_schema input_typespackagesreplace _suppress_local_package_warnings)rrrruse_user_schemaxml_reader_udtfrr  CACHERESULTzDataFrameReader.)8rrrrJ is_closedrC#LOCAL_TESTING_SUPPORTED_FILE_FORMATrrrrrrrrrrrNrrrrrr-r8r<r4STAGEr1rcollect upload_filer/r,osrtbasenamer*TABLE_FUNCTIONrLrOr.udtfregister_from_filerrBrrrrrr%r0r cache_result_all_variant_cols)rrtrrrrrrrdrr temp_stagesql_create_temp_stagepython_file_path handler_namerrrrs r}rz*DataFrameReader._read_semi_structured_fileOs! dmm))+? @}}"",,..X%@@ ##;;,A&*O*V%-s6{$; 3 <   6!9uVH,IJK K#D) FKM23$(!    001B1BC  )F05D  n -"O    224@  )#,0,Q,Q,S)/ U?1T5F5FF%'99M9MN *J:,VWXnWo(p% !!"75!IQQ#R ##//("'"15 0'3^JE #.TE   $ $ * ( "! "(# $"% &$:' ( ) .!++ 6~7K7KL &&", -eW -{m ' % <   . /22  ""i3 $$&7?*7L34*0--*E*E*Gh' --MM !  F$>? % .s I%z1.29.0irDF)rrrrrr max_workersrrrrrrfetch_merge_countfetch_with_processconnection_parametersrcreate_connection).rrrrrc|s|r|r |r tdt}t|d<|dnd|d<|xs|}|dnd}td i}t j }t jd || rt| tr| g} t|||||||| | | | | ||}|j}|j}|jj|d <|jj|d <|5d |vr t!dt#t$j&}|j(j+|Dcgc]}|gc}dgj,j/|d||j1|j(|||d | |j3dd|j3dd| | |d }t j }||z |d<t jd|d||z |j(j4j6j9|t;|t|St#t$j&}t#t$j<}t?|j(||||tA|j(||||||||| t j }||z |d<|jBjE||||} | jjG|d<|j(j4j6j9|t jd|d||z t jdtIjJ|t;| t| Scc}w)a-2 Reads data from a database table or query into a DataFrame using a DBAPI connection, with support for optional partitioning, parallel processing, and query customization. Usage Notes: - Ingestion performance tuning: - **Partitioning**: Use ``column``, ``lower_bound``, ``upper_bound``, and ``num_partitions`` together to split large tables into smaller partitions for parallel processing. All four parameters must be specified together, otherwise an error will be raised. - **Parallel execution**: Set ``max_workers`` to control the maximum number of processes and threads used for parallel execution. - **Fetch optimization**: Adjust ``fetch_size`` to optimize performance by reducing the number of round trips to the database. - **Partition filtering**: Use ``predicates`` to define WHERE conditions for partitions. Note that ``predicates`` will be ignored if ``column`` is specified for partitioning. - **Schema specification**: Set ``custom_schema`` to skip schema inference. The custom schema must have matching column names with the table in the external data source. - Execution timing and error handling: - **UDTF Ingestion**: Uses lazy evaluation. Errors are reported as ``SnowparkSQLException`` during DataFrame actions (e.g., ``DataFrame.collect()``). - **Local Ingestion**: Uses eager execution. Errors are reported immediately as ``SnowparkDataFrameReaderException`` when this method is called. Args: create_connection: A callable that returns a DB-API compatible database connection. The callable can optionally accept keyword arguments via `**kwargs`. If connection_parameters is provided, those will be passed as keyword arguments to this callable. The callable must be picklable, as it will be passed to and executed in child processes. table: The name of the table in the external data source. This parameter cannot be used together with the `query` parameter. query: A valid SQL query to be used as the data source in the FROM clause. This parameter cannot be used together with the `table` parameter. column: The column name used for partitioning the table. Partitions will be retrieved in parallel. The column must be of a numeric type (e.g., int or float) or a date type. When specifying `column`, `lower_bound`, `upper_bound`, and `num_partitions` must also be provided. lower_bound: lower bound of partition, decide the stride of partition along with `upper_bound`. This parameter does not filter out data. It must be provided when `column` is specified. upper_bound: upper bound of partition, decide the stride of partition along with `lower_bound`. This parameter does not filter out data. It must be provided when `column` is specified. num_partitions: number of partitions to create when reading in parallel from multiple processes and threads. It must be provided when `column` is specified. max_workers: number of processes and threads used for parallelism. query_timeout: The timeout (in seconds) for each query execution. A default value of `0` means the query will never time out. The timeout behavior can also be configured within the `create_connection` method when establishing the database connection, depending on the capabilities of the DBMS and its driver. fetch_size: The number of rows to fetch per batch from the external data source. This determines how many rows are retrieved in each round trip, which can improve performance for drivers with a low default fetch size. custom_schema: a custom snowflake table schema to read data from external data source, the column names should be identical to corresponded column names external data source. This can be a schema string, for example: "id INTEGER, int_col INTEGER, text_col STRING", or StructType, for example: StructType([StructField("ID", IntegerType(), False), StructField("INT_COL", IntegerType(), False), StructField("TEXT_COL", StringType(), False)]) predicates: A list of expressions suitable for inclusion in WHERE clauses, where each expression defines a partition. Partitions will be retrieved in parallel. If both `column` and `predicates` are specified, `column` takes precedence. session_init_statement: One or more SQL statements executed before fetching data from the external data source. This can be used for session initialization tasks such as setting configurations. For example, `"SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED"` can be used in SQL Server to avoid row locks and improve read performance. The `session_init_statement` is executed only once at the beginning of each partition read. udtf_configs: A dictionary containing configuration parameters for ingesting external data using a Snowflake UDTF. If this parameter is provided, the workload will be executed within a Snowflake UDTF context. The dictionary may include the following keys: - external_access_integration (str, required): The name of the external access integration, which allows the UDTF to access external endpoints. - imports (List[str], optional): A list of stage file names to import into the UDTF. Use this to include any private packages required by your `create_connection()` function. - packages (List[str], optional): A list of package names (with optional version numbers) required as dependencies for your `create_connection()` function. fetch_merge_count: The number of fetched batches to merge into a single Parquet file before uploading it. This improves performance by reducing the number of small Parquet files. Defaults to 1, meaning each `fetch_size` batch is written to its own Parquet file and uploaded separately. fetch_with_process: Whether to use multiprocessing for data fetching and Parquet file generation in local ingestion. Default to `False`, which means multithreading is used to fetch data in parallel. Setting this to `True` enables multiprocessing, which may improve performance for CPU-bound tasks like Parquet file generation. When using multiprocessing, guard your script with `if __name__ == "__main__":` and call `multiprocessing.freeze_support()` on Windows if needed. This parameter has no effect in UDFT ingestion. connection_parameters: Optional dictionary of parameters to pass to the create_connection callable. If provided, these parameters will be unpacked and passed as keyword arguments to create_connection(`**connection_parameters`). This allows for flexible connection configuration without hardcoding values in the callable. Example: {"timeout": 30, "isolation_level": "READ_UNCOMMITTED"} Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection df = session.read.dbapi(create_oracledb_connection, table=...) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection df = session.read.dbapi(create_oracledb_connection, table=..., fetch_with_process=True) Example:: .. code-block:: python import sqlite3 def create_sqlite_connection(timeout=5.0, isolation_level=None, **kwargs): connection = sqlite3.connect( database=":memory:", timeout=timeout, isolation_level=isolation_level ) return connection connection_params = {"timeout": 30.0, "isolation_level": "DEFERRED"} df = session.read.dbapi( create_sqlite_connection, table=..., connection_parameters=connection_params ) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection # pull data from target table with parallelism using partition column df_local_par_column = session.read.dbapi( create_oracledb_connection, table="target_table", fetch_size=100000, num_partitions=4, column="ID", # swap with the column you want your partition based on upper_bound=10000, lower_bound=0 ) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection # pull data from target table with parallelism using predicates df_local_predicates = session.read.dbapi( create_oracledb_connection, table="target_table", fetch_size=100000, predicates=[ "ID < 3", "ID >= 3" ] ) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection udtf_configs = { "external_access_integration": "" } # pull data from target table with udtf ingestion df_udtf_basic = session.read.dbapi( create_oracledb_connection, table="target_table", udtf_configs=udtf_configs ) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection udtf_configs = { "external_access_integration": "" } # pull data from target table with udtf ingestion with parallelism using partition column df_udtf_par_column = session.read.dbapi( create_oracledb_connection, table="target_table", udtf_configs=udtf_configs, fetch_size=100000, num_partitions=4, column="ID", # swap with the column you want your partition based on upper_bound=10000, lower_bound=0 ) Example:: .. code-block:: python import oracledb def create_oracledb_connection(): connection = oracledb.connect(...) return connection udtf_configs = { "external_access_integration": "" } # pull data from target table with udtf ingestion with parallelism using partition column df_udtf_predicates = session.read.dbapi( create_oracledb_connection, table="target_table", udtf_configs=udtf_configs, fetch_size=100000, predicates=[ "ID < 3", "ID >= 3" ] ) rrNrr rTFrrr driver_typerzexternal_access_integration cannot be None when udtf ingestion is used. Please refer to https://docs.snowflake.com/en/sql-reference/sql/create-external-access-integration to create external access integrationr[rr?rrr)rrrrrrrrzingestion end at: z, total time: )rrsnowflake_table_namesnowflake_stage_namer) r partitionerpartitioned_queriesrrrrrrrrrzrecorded telemetry: )&rDrrrrrrrrrrrrrrrrr<r4rrrrr_udtf_ingestionrrrrr%rrr driverrrrdumps)!rrrrrrrrrrrrrrrrrrrrrrrrr struct_schemarrrrrrrs! r}rzDataFrameReader.dbapisHe52K !, 1Lo. , 8 >O ./% =4e+G*M'&&(  +J<89 !j1G&M&<%= "+            "  ! $** )44-8-B-B-H-Hk*/:/F/F/L/Lm,  #,L@ g ;>;O;OP  MM * *&9:U%:K= + eMM !!@" ,,  :;%$((D9%))*d;'=+!@- B((*H;Cj;P !"7 8 LL$XJnX =R;O;OP:>;O;OP*MM !5!5,K   MM# 3#1!5!5,K"7 $$&7?*7L34 ##99 & : +1--*E*E*Gh' --MM !     .J9N8O P   +DJJ7L,M+NOPF$?@ Y;s+ M,z1.37.0c ^t|}d|}d|jvr"|jd}t|}|d|z }|jj ||j t ttdtdjd}t|d|S) a Returns a DataFrame with a single column ``FILE`` containing a list of files in the specified Snowflake stage location (either internal or external). Args: path: The stage location to list files from (e.g., "@mystage", "@mystage/path/"). Example:: >>> # Create a temp stage and upload some test files >>> _ = session.sql("CREATE OR REPLACE TEMP STAGE mystage").collect() >>> _ = session.file.put("tests/resources/testCSV.csv", "@mystage", auto_compress=False) >>> _ = session.file.put("tests/resources/testCSVheader.csv", "@mystage", auto_compress=False) >>> _ = session.file.put("tests/resources/testJson.json", "@mystage", auto_compress=False) >>> # List all files in the stage >>> df = session.read.file("@mystage") >>> df.count() 3 >>> # List files matching a pattern (only CSV files) >>> df = session.read.option("pattern", ".*\.csv").file("@mystage") >>> df.count() 2 >>> # List files with a more specific pattern >>> df = session.read.option("pattern", ".*header.*").file("@mystage") >>> df.count() 1 zLIST rbz PATTERN = r@z"name"filezDataFrameReader.file) rrrrrselectrIrGrHrFaliasr%)rrtr list_queryrorprs r}rzDataFrameReader.file%sB$D)TF^  )) )'' 2G9'BO K'89 9J ]]  zY  ? F F F3s8S]3 4 : :6 B  B 67 r stage_namecjd}|r||jp|jjj}t |j j |}|jj|j||_ |jj|d}|r||j|_ |S)ai Returns a DataFrame representing the results of a directory table query on the specified stage. This method is an alias of :meth:`~snowflake.snowpark.session.Session.directory`. Args: stage_name: The name of the stage to query. The stage name should not include the '@' prefix as it will be added automatically. Returns: A DataFrame containing metadata about files in the stage with the following columns: - ``RELATIVE_PATH``: Path to the files to access using the file URL - ``SIZE``: Size of the file in bytes - ``LAST_MODIFIED``: Timestamp when the file was last updated in the stage - ``MD5``: MD5 checksum for the file - ``ETAG``: ETag header for the file - ``FILE_URL``: Snowflake file URL to access the file NFr) rrrrrrread_directoryrrr directoryrr)rrrrr dataframes r}rzDataFrameReader.directory\s, .==++002D#DII$<$rrrrr}rrsvWrOS;HL . 1%S 4 ?8R2S1Bj ?C#' $48BF $jC#&'jj #7>#:; j C= j jE#x-01j!s,='=!>?j j jjXZDDU"HL &|4 AE    D# ^^%S%T%%"   K     ,)>#)>$)>))>)>Vggg ggR06CDI8 6RRR RRh%#%c%d%FW%%N@D~9= 0@K@K@K tT4* + @KDE%E tT4 E4IsICIIIVX& &*## $1515(,'($%:>*.BF#F FF TN F } F}F FeCHo.FeCHo.F! F }FSMF c:o 67FT#Y'F !)sDI~)> ?!F"#F$ %F'FPX& $# $1515(,%)'($*:>*.BF'+!"#(04)}#$56}} } } }  }eCHo.}eCHo.}! }c]} }}SM} c:o 67}T#Y'}!)sDI~)> ?} tn!}"#}$!%}& (~'}()}* +}'}~ (#3333$3j"C"D"I"rr)rrrUsysr collectionsrloggingrtypingrrrrr r r r r snowflake.snowpark snowflake4snowflake.snowpark._internal.proto.generated.ast_pb2snowpark _internalr generatedast_pb24snowflake.snowpark._internal.analyzer.analyzer_utilsrrrrr0snowflake.snowpark._internal.analyzer.expressionr9snowflake.snowpark._internal.analyzer.snowflake_plan_noder6snowflake.snowpark._internal.analyzer.unary_expressionr&snowflake.snowpark._internal.ast.utilsrrrr(snowflake.snowpark._internal.data_sourcer?snowflake.snowpark._internal.data_source.datasource_partitionerr:snowflake.snowpark._internal.data_source.datasource_typingr.snowflake.snowpark._internal.data_source.utilsrrrr r!r"r#*snowflake.snowpark._internal.error_messager$&snowflake.snowpark._internal.telemetryr%'snowflake.snowpark._internal.type_utilsr&r'r(r)&snowflake.snowpark._internal.udf_utilsr*"snowflake.snowpark._internal.utilsr+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>snowflake.snowpark.columnr?r@rAsnowflake.snowpark.dataframerBsnowflake.snowpark.exceptionsrCrDsnowflake.snowpark.functionsrErFrGrHrI#snowflake.snowpark.mock._connectionrJsnowflake.snowpark.tablerKsnowflake.snowpark.typesrLrMrNrO version_inforRcollections.abcrrrrr-_MAX_RETRY_TIMErrr rrrrr}r/s{ #MMMDDDGRH :RWF M,TS2MLD*v( 8 &/#+& ## ! ! +) ss!t!!HNNr