Complete Workflows

# Find population datasets from Vienna
results = search_datasets(
  query="Bevölkerung Wien Bezirk",
  themes=["SOCI"],
  formats=["CSV"],
  publishers=["Stadt Wien"],
  boost_quality=True,
  limit=10
)

# Display results count
print(f"Found {results['count']} datasets")

# Review top 3 results
for i, dataset in enumerate(results['results'][:3], 1):
  title = dataset.get('title', {}).get('de', 'No title')
  print(f"{i}. {title}")

# Pick the first result
dataset_id = results['results'][0]['id']

# Get complete metadata
dataset = get_dataset(dataset_id=dataset_id)

# Review key information
print(f"Title: {dataset['title']['de']}")
print(f"Description: {dataset['description']['de'][:200]}...")
print(f"Modified: {dataset['modified']}")
print(f"Publisher: {dataset['publisher']['name']}")
print(f"License: {dataset.get('license', 'Not specified')}")

# Retrieve all distributions
distributions = get_dataset_distributions(dataset_id=dataset_id)

# Find CSV distribution
csv_dist = next(
  (d for d in distributions
   if d.get('format', {}).get('id') == 'CSV'),
  None
)

if csv_dist:
  download_url = csv_dist['access_url']
  file_size = csv_dist.get('byte_size', 'Unknown')
  print(f"Download URL: {download_url}")
  print(f"File size: {file_size} bytes")

# Check schema first
schema = preview_schema(url=download_url, format="CSV")

print("Columns:")
for col in schema['columns']:
  print(f"  - {col['name']}: {col['type']}")

# Preview sample data
preview = preview_data(url=download_url, max_rows=20, format="CSV")

print(f"\nShowing {preview['row_count']} sample rows:")
for row in preview['data'][:5]:
  print(row)

# Verify structure matches expectations
expected_columns = ["Jahr", "Bezirk", "Einwohner"]
actual_columns = [c['name'] for c in schema['columns']]

if all(col in actual_columns for col in expected_columns):
  print("Success: Schema validated")
  # Proceed with full download using download_url
else:
  print("Error: Schema mismatch, check dataset")

# Use semantic search for natural language
results = semantic_search_datasets(
  natural_query="Luftqualität Messungen österreichische Städte",
  limit=5
)

print(f"Found {len(results['results'])} highly relevant datasets")

# Display semantic search results
for i, dataset in enumerate(results['results'], 1):
  title = dataset.get('title', {}).get('de', 'No title')
  categories = [c.get('id') for c in dataset.get('categories', [])]
  print(f"{i}. {title}")
  print(f"   Categories: {', '.join(categories)}")

# Pick most relevant dataset
dataset_id = results['results'][0]['id']

# Get quality analysis
quality = analyze_dataset_quality(dataset_id=dataset_id)

print(f"Quality Score: {quality['score']}/100")
print(f"Completeness: {quality['completeness']}")
print("\nQuality Components:")
for component, value in quality['components'].items():
  status = "✓" if value else "✗"
  print(f"  {status} {component}")

# Discover similar datasets
related = find_related_datasets(
  dataset_id=dataset_id,
  limit=5,
  min_score=30
)

print(f"\nFound {len(related['results'])} related datasets:")

for dataset in related['results']:
  title = dataset.get('title', {}).get('de', 'No title')
  score = dataset.get('match_score', 0)
  themes = [t.get('id') for t in dataset.get('themes', [])]

  print(f"Score {score}: {title}")
  print(f"  Shared themes: {', '.join(themes)}")

# Compare original and related dataset
original = get_dataset(dataset_id=dataset_id)
related_id = related['results'][0]['id']
related_dataset = get_dataset(dataset_id=related_id)

# Compare key attributes
print("\nComparison:")
print(f"Original Publisher: {original['publisher']['name']}")
print(f"Related Publisher: {related_dataset['publisher']['name']}")

original_themes = {c['id'] for c in original.get('categories', [])}
related_themes = {c['id'] for c in related_dataset.get('categories', [])}
shared_themes = original_themes & related_themes

print(f"Shared themes: {', '.join(shared_themes)}")

# Search with quality boost
results = search_datasets(
  query="Arbeitsmarkt Beschäftigung",
  themes=["ECON"],
  formats=["CSV", "JSON"],
  boost_quality=True,
  sort_by="modified_desc",
  limit=20
)

print(f"Found {results['count']} datasets")

# Analyze each dataset
high_quality_datasets = []

for dataset in results['results'][:10]:
  dataset_id = dataset['id']

  # Get quality analysis
  quality = analyze_dataset_quality(dataset_id=dataset_id)

  # Filter: score > 70 and critical components present
  if quality['score'] > 70:
    critical_ok = all([
      quality['components']['has_title'],
      quality['components']['has_description'],
      quality['components']['has_publisher'],
      quality['components']['has_license']
    ])

    if critical_ok:
      high_quality_datasets.append({
        'id': dataset_id,
        'title': dataset.get('title', {}).get('de', 'No title'),
        'score': quality['score'],
        'modified': dataset.get('modified')
      })

print(f"Found {len(high_quality_datasets)} high-quality datasets")

# Ensure data is accessible
accessible_datasets = []

for dataset in high_quality_datasets:
  # Get distributions
  distributions = get_dataset_distributions(dataset_id=dataset['id'])

  # Check for valid download URLs
  has_csv = any(
    d.get('format', {}).get('id') == 'CSV' and d.get('access_url')
    for d in distributions
  )

  if has_csv:
    # Verify URL is accessible
    csv_url = next(
      d['access_url'] for d in distributions
      if d.get('format', {}).get('id') == 'CSV'
    )

    try:
      schema = preview_schema(url=csv_url, format="CSV")

      accessible_datasets.append({
        **dataset,
        'url': csv_url,
        'columns': len(schema['columns'])
      })

      print(f"Success: {dataset['title']}")
      print(f"  Score: {dataset['score']}, Columns: {len(schema['columns'])}")

    except Exception as e:
      print(f"Error: {dataset['title']} - URL not accessible")

print(f"\n{len(accessible_datasets)} datasets ready for use")

# German semantic search
de_results = semantic_search_datasets(
  natural_query="Gesundheitsdaten aus Wien",
  limit=10
)

print("German query results:")
for dataset in de_results['results'][:3]:
  print(f"- {dataset['title']['de']}")

# English semantic search
en_results = semantic_search_datasets(
  natural_query="health data from Vienna",
  limit=10
)

print("\nEnglish query results:")
for dataset in en_results['results'][:3]:
  print(f"- {dataset['title'].get('de', 'No title')}")

# Extract dataset IDs
de_ids = {d['id'] for d in de_results['results']}
en_ids = {d['id'] for d in en_results['results']}

# Find overlap
overlap = de_ids & en_ids
print(f"\nOverlapping datasets: {len(overlap)}/{len(de_ids)}")

# Check language detection effectiveness
overlap_pct = (len(overlap) / len(de_ids)) * 100
if overlap_pct > 70:
  print(f"Success: Language detection effective ({overlap_pct:.0f}% overlap)")

# Get all catalogues
catalogues = list_catalogues(limit=50)

# Find Vienna catalogue
vienna = next(
  c for c in catalogues
  if 'wien' in c.get('title', {}).get('de', '').lower()
)

print(f"Found: {vienna['title']['de']}")
print(f"Catalogue ID: {vienna['id']}")
print(f"Dataset count: {vienna['count']}")

# Get detailed catalogue information
catalogue = get_catalogue(catalogue_id=vienna['id'])

print(f"\nCatalogue: {catalogue['title']['de']}")
print(f"Description: {catalogue.get('description', {}).get('de', 'N/A')[:200]}")
print(f"Datasets: {catalogue['count']}")
print(f"Modified: {catalogue['modified']}")

# Search within catalogue
datasets = search_datasets(
  query="",
  catalogues=[vienna['id']],
  sort_by="modified_desc",
  limit=20
)

print(f"\nRecent datasets from {vienna['title']['de']}:")

for i, dataset in enumerate(datasets['results'][:10], 1):
  title = dataset['title']['de']
  modified = dataset['modified']
  categories = [c['id'] for c in dataset.get('categories', [])]

  print(f"{i}. {title}")
  print(f"   Modified: {modified}, Categories: {', '.join(categories)}")

# Analyze category coverage
category_counts = {}

for dataset in datasets['results']:
  for category in dataset.get('categories', []):
    cat_id = category['id']
    category_counts[cat_id] = category_counts.get(cat_id, 0) + 1

print("\nCategory distribution:")
for cat, count in sorted(
  category_counts.items(),
  key=lambda x: x[1],
  reverse=True
):
  print(f"  {cat}: {count} datasets")