Skip to content
Snippets Groups Projects
Commit 75241865 authored by Nikhil Lobo's avatar Nikhil Lobo
Browse files

beam assignment

parent d14cd382
Branches master
No related tags found
No related merge requests found
import apache_beam as beam
table_schema = 'column1:STRING, column2:STRING'
table_spec = "[project_id]:[dataset_id].[table_id]"
with beam.Pipeline() as pipeline:
ip = ( pipeline
|beam.io.ReadFromText("gs://path-to-file",skip_header_lines=True)
|beam.Map(lambda x : x.split(","))
|beam.Map(lambda y : {'column1': y[0],'column2': y[1]})
|beam.io.WriteToBigQuery(
table_spec,
schema=table_schema,
write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED)
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment