Update scraping spark example image

So that it's clearer to see what's going on.
DAGWorks-Inc · Mar 5, 2024 · 66cc3ba · 66cc3ba
1 parent 08e9f28
commit 66cc3ba
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/pipeline.png b/examples/LLM_Workflows/scraping_and_chunking/spark/pipeline.png
diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py b/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py
@@ -76,7 +76,11 @@ def chunked_url_text(urls_from_sitemap: ps.DataFrame) -> ps.DataFrame:
  from hamilton import driver
 
  dr = driver.Builder().with_modules(doc_pipeline, spark_pipeline).with_config({}).build()
- dr.display_all_functions("pipeline.png")
+ dr.visualize_execution(
+ ["chunked_url_text"],
+ output_file_path="pipeline.png",
+ inputs={"app_name": "chunking_spark_job", "num_partitions": 4},
+ )
  result = dr.execute(
  ["chunked_url_text"],
  inputs={"app_name": "chunking_spark_job", "num_partitions": 4},