Skip to contents

Example: Structured Data Extraction Pipeline

Source: vignettes/example-extraction-pipeline.Rmd
example-extraction-pipeline.Rmd

This example builds a pipeline that extracts structured metadata from an R package’s source files. Each step is a separate agent call, with the output of one step feeding into the next.

Use separate steps when you need to inspect or validate an intermediate result before continuing. Here, you can review the extracted metadata before asking for categories and a report.

The Pipeline

Our pipeline has three steps:

  1. Extract – Read package files and extract raw metadata as JSON
  2. Enrich – Categorise and annotate the extracted metadata
  3. Report – Produce a human-readable summary

The first two steps use ellmer types for structured results. The final step returns a report as text.

Step 1: Extract Package Metadata

The first agent reads the DESCRIPTION file and key R source files, then extracts structured information:

library(deputy)

extract_schema <- list(
  type = "object",
  properties = list(
    name = list(type = "string"),
    title = list(type = "string"),
    version = list(type = "string"),
    authors = list(
      type = "array",
      items = list(
        type = "object",
        properties = list(
          name = list(type = "string"),
          role = list(type = "string")
        ),
        required = c("name", "role")
      )
    ),
    dependencies = list(
      type = "array",
      items = list(type = "string")
    ),
    exported_functions = list(
      type = "array",
      items = list(
        type = "object",
        properties = list(
          name = list(type = "string"),
          file = list(type = "string")
        ),
        required = c("name")
      )
    )
  ),
  required = c("name", "title", "version", "authors", "dependencies",
    "exported_functions")
)
chat <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")

extractor <- Agent$new(
  chat = chat,
  tools = tools_file(),
  permissions = permissions_readonly(),
  system_prompt = "You are a metadata extractor. Read the package
    DESCRIPTION file and scan R/ source files for exported functions
    (look for @export roxygen tags). Return structured JSON only."
)

step1 <- extractor$run_sync(
  "Extract metadata from this R package. Read DESCRIPTION and scan
   the R/ directory for exported functions.",
  type = ellmer::type_from_schema(jsonlite::toJSON(extract_schema, auto_unbox = TRUE))
)

if (!result_is_success(step1)) {
  cli::cli_abort("Extraction stopped early: {step1$stop_reason}.")
}

pkg_metadata <- step1$structured_output
pkg_metadata$name
pkg_metadata$version
length(pkg_metadata$exported_functions)

Step 2: Enrich with Categorisation

The second agent takes the extracted JSON and adds categorisation and summaries:

enrich_schema <- list(
  type = "object",
  properties = list(
    name = list(type = "string"),
    title = list(type = "string"),
    version = list(type = "string"),
    category_tags = list(
      type = "array",
      items = list(
        type = "string",
        enum = c("data", "modeling", "visualization", "infrastructure",
          "testing", "io", "web", "cli")
      )
    ),
    complexity = list(
      type = "string",
      enum = c("simple", "moderate", "complex")
    ),
    functions = list(
      type = "array",
      items = list(
        type = "object",
        properties = list(
          name = list(type = "string"),
          summary = list(type = "string")
        ),
        required = c("name", "summary")
      )
    )
  ),
  required = c("name", "title", "version", "category_tags",
    "complexity", "functions")
)
chat2 <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")

enricher <- Agent$new(
  chat = chat2,
  tools = tools_file(),
  permissions = permissions_readonly(),
  system_prompt = "You are a package analyst. Given package metadata,
    categorise the package, assess its complexity, and write a one-line
    summary of each exported function. You may read source files to
    understand what functions do. Return structured JSON only."
)

# Pass the extracted metadata as context
step2 <- enricher$run_sync(
  paste(
    "Enrich this package metadata with categorisation and function summaries.",
    "Read the source files if needed to understand what functions do.",
    "",
    "Package metadata:",
    jsonlite::toJSON(pkg_metadata, auto_unbox = TRUE, pretty = TRUE)
  ),
  type = ellmer::type_from_schema(jsonlite::toJSON(enrich_schema, auto_unbox = TRUE))
)

if (!result_is_success(step2)) {
  cli::cli_abort("Enrichment stopped early: {step2$stop_reason}.")
}
enriched <- step2$structured_output
enriched$category_tags
enriched$complexity

Step 3: Generate Report

The third agent takes the enriched data and produces a readable report:

chat3 <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")

reporter <- Agent$new(
  chat = chat3,
  system_prompt = "You are a technical writer. Given enriched package
    metadata, produce a clear, concise package summary report in
    markdown format. Include sections for overview, key functions,
    dependencies, and recommendations."
)

step3 <- reporter$run_sync(
  paste(
    "Write a package summary report from this enriched metadata:",
    jsonlite::toJSON(enriched, auto_unbox = TRUE, pretty = TRUE)
  )
)

cat(step3$response)

Putting It Together

Wrap the pipeline in a reusable function:

extraction_pipeline <- function(package_dir = ".") {
  # Step 1: Extract
  chat1 <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")
  extractor <- Agent$new(
    chat = chat1,
    tools = tools_file(),
    permissions = permissions_readonly(),
    working_dir = package_dir,
    system_prompt = "Extract package metadata. Read DESCRIPTION and
      scan R/ for exported functions. Return structured JSON only."
  )

  step1 <- extractor$run_sync(
    "Extract metadata from this R package.",
    type = ellmer::type_from_schema(jsonlite::toJSON(extract_schema, auto_unbox = TRUE))
  )

  if (!result_is_success(step1)) {
    cli::cli_abort("Extraction stopped early: {step1$stop_reason}.")
  }

  # Step 2: Enrich
  chat2 <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")
  enricher <- Agent$new(
    chat = chat2,
    tools = tools_file(),
    permissions = permissions_readonly(),
    working_dir = package_dir,
    system_prompt = "Categorise and summarise package functions.
      Return structured JSON only."
  )

  step2 <- enricher$run_sync(
    paste("Enrich this metadata:", jsonlite::toJSON(
      step1$structured_output,
      auto_unbox = TRUE
    )),
    type = ellmer::type_from_schema(jsonlite::toJSON(enrich_schema, auto_unbox = TRUE))
  )

  if (!result_is_success(step2)) {
    cli::cli_abort("Enrichment stopped early: {step2$stop_reason}.")
  }

  # Step 3: Report
  chat3 <- ellmer::chat_anthropic(model = "claude-sonnet-4-20250514")
  reporter <- Agent$new(chat = chat3)

  step3 <- reporter$run_sync(
    paste("Write a package summary:", jsonlite::toJSON(
      step2$structured_output,
      auto_unbox = TRUE, pretty = TRUE
    ))
  )

  costs <- c(step1$cost$total, step2$cost$total, step3$cost$total)

  list(
    metadata = step1$structured_output,
    enriched = step2$structured_output,
    report = step3$response,
    total_cost = if (anyNA(costs)) NA_real_ else sum(costs)
  )
}

# Usage
result <- extraction_pipeline(".")
cat(result$report)

Error Handling Between Steps

Check is_success() before passing a result to the next step. Use validate for application rules; invalid output or an unavailable validator signals an error, and bounded corrections are opt-in:

step <- extractor$run_sync(
  "Extract metadata.",
  type = ellmer::type_from_schema(jsonlite::toJSON(extract_schema, auto_unbox = TRUE))
)

if (!result_is_success(step)) {
  cli::cli_abort("Agent stopped early: {step$stop_reason}")
}

# Safe to use the parsed output
parsed <- step$structured_output

Next Steps