""" Views for the Atlas Record Cleaner Report application. """ import csv import io import json import logging import random from datetime import datetime, timedelta from typing import Dict, Any, List, Optional from django.shortcuts import render from django.http import JsonResponse, HttpResponse from django.views.decorators.http import require_http_methods from django.core.cache import cache from django.conf import settings from components.utils import format_atlas_event_date from .services.atlas_service import fetch_data_resources, fetch_atlas_occurrences, DEFAULT_FIELDS from .services.record_cleaner_service import RecordCleanerClient, generate_summary_report from .services.data_mapper import map_occurrence_to_record_cleaner, merge_validation_and_verification_results from .services.csv_service import generate_csv_report logger = logging.getLogger(__name__) # Cache configuration CACHE_TIMEOUT = 1200 # 20 minutes in seconds CACHE_KEY_DATASETS = 'atlas_datasets_list' def _generate_records_warning(original_count: int, filtered_out_count: int, processed_count: int, max_records: int) -> Optional[str]: """ Generate a warning message if fewer records were processed than expected. Args: original_count: Total number of records fetched from Atlas filtered_out_count: Number of records filtered out (e.g., missing eventDate) processed_count: Number of records actually processed max_records: Maximum number of records expected Returns: Warning message string, or None if no warning needed """ if processed_count >= max_records: return None reasons = [] if original_count == 0: reasons.append("No records found matching your criteria. ") else: if original_count < max_records: record_word = "record" if original_count == 1 else "records" reasons.append(f"Only {original_count} {record_word} were in the dataset/search results. ") if filtered_out_count > 0: record_word = "record" if filtered_out_count == 1 else "records" reasons.append(f"{filtered_out_count} {record_word} were excluded because they lack an event date. This is not a Record Cleaner limitation, it's just a simplification requirement for the demo.") if reasons: return "" + " ".join(reasons) return None def generate_message_summary( merged_records: List[Dict[str, Any]], message_type: str, verification_result_filter: Optional[str] = None ) -> List[Dict[str, Any]]: """ Generate a summary of validation or verification messages. Args: merged_records: List of merged records with validation/verification data message_type: Either 'validation' or 'verification' verification_result_filter: Optional filter by verification_result ('fail', 'warn', 'pass', or None for all) Returns: List of dictionaries with 'message', 'count', and 'record_ids' keys, sorted by count descending """ message_field = f'{message_type}_messages' message_data = {} # message -> {'count': int, 'record_ids': list} # Collect all messages from records for record in merged_records: # Filter by verification_result if specified if verification_result_filter: record_verification_result = record.get('verification_result', '') # Treat 'not_verified' as 'fail' for filtering purposes if record_verification_result == 'not_verified' and verification_result_filter == 'fail': pass # Include not_verified records in fail group elif record_verification_result != verification_result_filter: continue record_message_list = record.get(message_field, '') # Track which records have each message record_id = record.get('occurrenceID', '') for message in record_message_list: if message not in message_data: message_data[message] = {'count': 0, 'record_ids': []} message_data[message]['count'] += 1 if record_id and record_id not in message_data[message]['record_ids']: message_data[message]['record_ids'].append(record_id) # Convert to list of dictionaries and sort by count summary = [ { 'message': msg, 'count': data['count'], 'record_ids': data['record_ids'] } for msg, data in message_data.items() ] summary.sort(key=lambda x: x['count'], reverse=True) return summary @require_http_methods(["GET"]) def index(request): """ Display the Record Cleaner report form. """ # Get datasets from cache or fetch fresh datasets = cache.get(CACHE_KEY_DATASETS) if datasets is None: try: datasets = fetch_data_resources() datasets.sort(key=lambda x: x.get('name', '')) # Cache for 20 minutes cache.set(CACHE_KEY_DATASETS, datasets, CACHE_TIMEOUT) except Exception as e: logger.error(f"Failed to fetch datasets: {str(e)}", exc_info=True) # If fetch fails, use empty list and show error datasets = [] context = { 'datasets': datasets, 'error': f'Failed to load datasets. Perhaps the NBN Atlas Registry is a bit slow at the moment. Try again later.' } return render(request, 'atlas_record_cleaner_report/form.html', context) # Sort datasets by name for better UX datasets_sorted = sorted(datasets, key=lambda x: x.get('name', '')) context = { 'datasets': datasets_sorted, 'RECORD_CLEANER_MAX_RECORDS': settings.RECORD_CLEANER_MAX_RECORDS } return render(request, 'atlas_record_cleaner_report/form.html', context) @require_http_methods(["POST"]) def generate_report(request): """ Generate Record Cleaner report from Atlas data. """ # Get form data input_type = request.POST.get('input_type') data_resource_uid = request.POST.get('data_resource_uid') dataset_name = request.POST.get('dataset_name', '').strip() search_url = request.POST.get('search_url', '').strip() # Validate input if not input_type or input_type not in {'dataset', 'url'}: return JsonResponse({'error': 'Input type is required'}, status=400) if input_type == 'dataset' and not data_resource_uid: return JsonResponse({'error': 'Dataset selection is required'}, status=400) if input_type == 'url' and not search_url: return JsonResponse({'error': 'Search URL is required'}, status=400) try: # Step 1: Fetch occurrences from Atlas if input_type == 'dataset': occurrences = fetch_atlas_occurrences( data_resource_uid=data_resource_uid ) else: # url occurrences = fetch_atlas_occurrences( search_url=search_url ) if not occurrences: context = { 'error': 'No records found matching your criteria.', 'occurrences_count': 0 } return render(request, 'atlas_record_cleaner_report/results.html', context) # Track original count and filter out records without eventDate original_count = len(occurrences) occurrences = [occurrence for occurrence in occurrences if occurrence.get('eventDate')] filtered_out_count = original_count - len(occurrences) # Step 2: Store original records for debugging (before any processing) request.session['original_records'] = occurrences # Step 3: Map occurrences to Record Cleaner format record_cleaner_records = [] for idx, occurrence in enumerate(occurrences, start=1): rc_record = map_occurrence_to_record_cleaner(occurrence, idx) record_cleaner_records.append(rc_record) # Step 4: Store Record Cleaner JSON for debugging (before API call) request.session['record_cleaner_json'] = record_cleaner_records # Step 5: Initialize Record Cleaner client rc_client = RecordCleanerClient( username=settings.RECORD_CLEANER_USERNAME, password=settings.RECORD_CLEANER_PASSWORD ) # Step 6: Validate and verify records (API call - may fail, debug data already stored) validated, verified = rc_client.validate_and_verify(record_cleaner_records) # Step 7: Create lookup dictionaries for merging validated_by_id = {r['id']: r for r in validated} verified_by_id = {r['id']: r for r in verified} # Step 8: Merge results with original occurrences merged_records = [] for idx, occurrence in enumerate(occurrences, start=1): merged = merge_validation_and_verification_results( occurrence, validated_by_id.get(idx), verified_by_id.get(idx) ) merged_records.append(merged) merged_records.sort( key=lambda r: { "fail": 0, "warn": 1, "pass": 2 }.get(r.get("verification_result"), 2) ) # Step 9: Generate summary statistics summary = generate_summary_report(validated, verified) # Step 10: Generate message summaries separately for fail, warn, and pass # For fail and warn, exclude difficulty messages and track record IDs validation_message_summary_fail = generate_message_summary( merged_records, 'validation', verification_result_filter='fail' ) validation_message_summary_warn = generate_message_summary( merged_records, 'validation', verification_result_filter='warn' ) validation_message_summary_pass = generate_message_summary( merged_records, 'validation', verification_result_filter='pass' ) verification_message_summary_fail = generate_message_summary( merged_records, 'verification', verification_result_filter='fail' ) verification_message_summary_warn = generate_message_summary( merged_records, 'verification', verification_result_filter='warn' ) verification_message_summary_pass = generate_message_summary( merged_records, 'verification', verification_result_filter='pass' ) # Enrich message summaries with full record data for modal display def enrich_message_summary_with_records(message_summary, all_records, message_type): """Add full record data to message summary for modal display.""" for item in message_summary: item['records'] = [] for record_id in item.get('record_ids', []): for record in all_records: if record.get('occurrenceID') == record_id: item['records'].append(record) break return message_summary # Enrich summaries for modal validation_message_summary_fail = enrich_message_summary_with_records( validation_message_summary_fail, merged_records, 'validation' ) validation_message_summary_warn = enrich_message_summary_with_records( validation_message_summary_warn, merged_records, 'validation' ) verification_message_summary_fail = enrich_message_summary_with_records( verification_message_summary_fail, merged_records, 'verification' ) verification_message_summary_warn = enrich_message_summary_with_records( verification_message_summary_warn, merged_records, 'verification' ) # Step 11: Filter and group records to only show fail and warn on screen (pass records available in CSV) # Include records that failed validation (not_verified) as they are effectively failures fail_warn_records = [ record for record in merged_records if record.get('verification_result') in ['fail', 'warn', 'not_verified'] ] # Group records by category for display # Priority: Validation Fail > Validation Warn > Rules Check Fail > Rules Check Warn # This prevents records from appearing in multiple sections validation_fail_records = [ record for record in fail_warn_records if record.get('validation_result') == 'fail' ] # Validation warn records (only if validation didn't fail) validation_warn_records = [ record for record in fail_warn_records if record.get('validation_result') == 'warn' ] # Rules check fail records (only if validation passed) verification_fail_records = [ record for record in fail_warn_records if record.get('verification_result') == 'fail' and record.get('validation_result') == 'pass' ] # Rules check warn records (only if validation passed and verification didn't fail) verification_warn_records = [ record for record in fail_warn_records if record.get('verification_result') == 'warn' and record.get('validation_result') == 'pass' ] # Step 12: Store merged results in session for CSV download (all records) request.session['report_data'] = merged_records # Step 13: Check if fewer records processed than expected and prepare message processed_count = len(occurrences) records_warning = _generate_records_warning( original_count, filtered_out_count, processed_count, settings.RECORD_CLEANER_MAX_RECORDS ) # Step 14: Prepare context for template context = { 'occurrences_count': processed_count, 'summary': summary, 'validation_message_summary_fail': validation_message_summary_fail, 'validation_message_summary_warn': validation_message_summary_warn, 'validation_message_summary_pass': validation_message_summary_pass, 'verification_message_summary_fail': verification_message_summary_fail, 'verification_message_summary_warn': verification_message_summary_warn, 'verification_message_summary_pass': verification_message_summary_pass, 'merged_records': fail_warn_records, # Only fail/warn records for display 'validation_fail_records': validation_fail_records, 'validation_warn_records': validation_warn_records, 'verification_fail_records': verification_fail_records, 'verification_warn_records': verification_warn_records, 'input_type': input_type, 'data_resource_uid': data_resource_uid if input_type == 'dataset' else None, 'dataset_name': dataset_name if input_type == 'dataset' and dataset_name else None, 'search_url': search_url if input_type == 'url' else None, 'records_warning': records_warning, } return render(request, 'atlas_record_cleaner_report/results.html', context) except Exception as e: logger.error(f"Error generating report: {str(e)}", exc_info=True) # Check if we have stored debug data in session (before the error occurred) occurrences = request.session.get('original_records', []) context = { 'error': f'{str(e)}', 'occurrences_count': len(occurrences) } return render(request, 'atlas_record_cleaner_report/results.html', context) @require_http_methods(["GET"]) def download_csv(request): """ Download CSV export of the most recent report. """ merged_records = request.session.get('report_data') if not merged_records: return HttpResponse('No report data available. Please generate a report first.', status=404) return generate_csv_report(merged_records) @require_http_methods(["GET"]) def download_original_csv(request): """ Download CSV of original records (before Record Cleaner processing). For debugging purposes. """ occurrences = request.session.get('original_records') if not occurrences: return HttpResponse('No records to export', content_type='text/plain') # Create CSV in memory output = io.StringIO() # Use DEFAULT_FIELDS as the field ordering (occurrences are already normalized) fieldnames = list(DEFAULT_FIELDS) # Convert eventDate values if needed processed_records = [] for record in occurrences: processed_record = record.copy() if processed_record.get('eventDate'): processed_record['eventDate'] = format_atlas_event_date(processed_record['eventDate']) if processed_record.get('eventDateEnd'): processed_record['eventDateEnd'] = format_atlas_event_date(processed_record['eventDateEnd']) processed_records.append(processed_record) writer = csv.DictWriter(output, fieldnames=fieldnames) writer.writeheader() writer.writerows(processed_records) # Create HTTP response response = HttpResponse(output.getvalue(), content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="original_records.csv"' return response @require_http_methods(["GET"]) def download_record_cleaner_json(request): """ Download JSON payload that was sent to Record Cleaner. For debugging purposes. """ record_cleaner_records = request.session.get('record_cleaner_json') if not record_cleaner_records: return HttpResponse('No Record Cleaner JSON available. Please generate a report first.', status=404) # Create JSON response json_data = json.dumps(record_cleaner_records, indent=2) response = HttpResponse(json_data, content_type='application/json') response['Content-Disposition'] = 'attachment; filename="record_cleaner_payload.json"' return response @require_http_methods(["GET"]) def dummy_report(request): """ Generate a dummy report with test data for template development. """ # Species list for realistic data species_list = [ "Turdus merula", "Parus major", "Erithacus rubecula", "Fringilla coelebs", "Columba palumbus", "Corvus corone", "Pica pica", "Sturnus vulgaris", "Troglodytes troglodytes", "Carduelis carduelis", "Apodemus sylvaticus", "Sciurus carolinensis", "Vulpes vulpes", "Pipistrellus pipistrellus", "Bufo bufo" ] # Sample validation messages by result type validation_messages_fail = [ "Latitude invalid", "Longitude invalid", "Species not found in UK checklist", "Date format invalid", ] validation_messages_warn = [ "Coordinate uncertainty high", "Date precision low", "Species name requires verification", ] validation_messages_pass = [ "Date format verified", "Coordinates valid", ] # Sample verification messages by result type verification_messages_fail = [ "Failed: Species not recorded in this region before", "Failed: Outside known species range", "Failed: Phenology mismatch - species not expected at this time of year", ] verification_messages_warn = [ "Warning: Unusual for this habitat - review recommended", "Warning: Rare species - expert verification needed", "Warning: Unusual location for this species", ] verification_messages_pass = [ "Plausible identification for location and date", "Identification confirmed by expert", "Phenology matches expected for species", ] # Generate dummy records with deterministic distribution # Target: ~70% pass, ~20% warn, ~10% fail for validation num_records = 25 merged_records = [] # Pre-determine the distributions validation_distribution = ['pass'] * 18 + ['warn'] * 5 + ['fail'] * 2 # 25 records total random.shuffle(validation_distribution) # Pre-determine verification distribution for records that can be verified # Out of 23 records (18 pass + 5 warn validation), we want: # ~17 pass verification, ~4 warn verification, ~2 fail verification verification_distribution = ['pass'] * 17 + ['warn'] * 4 + ['fail'] * 2 random.shuffle(verification_distribution) verification_index = 0 # Track position in verification distribution for i in range(1, num_records + 1): # Random date within last year days_ago = random.randint(0, 365) occurrence_date = (datetime.now() - timedelta(days=days_ago)).strftime("%Y-%m-%d") # Random UK coordinates (roughly) latitude = random.uniform(50.0, 58.0) longitude = random.uniform(-5.0, 2.0) # Get validation result from predetermined distribution validation_result = validation_distribution[i - 1] # Assign validation message based on result if validation_result == 'fail': validation_msg = random.choice(validation_messages_fail) elif validation_result == 'warn': validation_msg = random.choice(validation_messages_warn) else: # pass validation_msg = random.choice(validation_messages_pass) if random.random() > 0.5 else "" # Verification only happens if validation passed or warned if validation_result in ["pass", "warn"]: # Get verification result from predetermined distribution verification_result = verification_distribution[verification_index] if verification_index < len(verification_distribution) else 'pass' verification_index += 1 # Assign verification message based on result if verification_result == 'fail': verification_msg = random.choice(verification_messages_fail) elif verification_result == 'warn': verification_msg = random.choice(verification_messages_warn) else: # pass verification_msg = random.choice(verification_messages_pass) if random.random() > 0.4 else "" id_difficulty = random.randint(1, 5) else: verification_result = 'not_verified' verification_msg = "" id_difficulty = None record = { 'occurrenceID': f'urn:{i:08d}', 'scientificName': random.choice(species_list), 'taxonConceptID': f'NHMSYS{i:06d}', 'eventDate': occurrence_date, 'decimalLatitude': latitude, 'decimalLongitude': longitude, 'coordinateUncertaintyInMeters': random.randint(10, 1000), 'lifeStage': random.choice(['adult', 'juvenile', 'larva', 'pupa', 'imago']), 'dataResourceID': 'DUMMY_DATA_RESOURCE', 'gridReference': f'NT{random.randint(10, 99)}{random.randint(10, 99)}', 'validation_result': validation_result, 'verification_result': verification_result, 'validation_messages': [validation_msg] if validation_msg else [], 'verification_messages': [verification_msg] if verification_msg else [], 'id_difficulty': id_difficulty, # Additional fields that might be in CSV 'recorder': random.choice(['John Smith', 'Jane Doe', 'Bob Wilson', 'Alice Brown']), } merged_records.append(record) # Calculate summary statistics validation_counts = {'pass': 0, 'warn': 0, 'fail': 0} verification_counts = {'pass': 0, 'warn': 0, 'fail': 0} not_verified_count = 0 for record in merged_records: validation_counts[record['validation_result']] += 1 verification_result = record.get('verification_result') if verification_result and verification_result != 'not_verified': verification_counts[verification_result] += 1 else: not_verified_count += 1 validation_total = sum(validation_counts.values()) validation_pass_rate = round((validation_counts['pass'] / validation_total * 100), 1) if validation_total > 0 else 0 verification_total = sum(verification_counts.values()) verification_pass_rate = round((verification_counts['pass'] / verification_total * 100), 1) if verification_total > 0 else 0 summary = { 'validation': validation_counts, 'verification': verification_counts, 'validation_pass_rate': validation_pass_rate, 'verification_pass_rate': verification_pass_rate, 'not_verified_count': not_verified_count } # Generate message summaries separately for fail, warn, and pass validation_message_summary_fail = generate_message_summary( merged_records, 'validation', verification_result_filter='fail' ) validation_message_summary_warn = generate_message_summary( merged_records, 'validation', verification_result_filter='warn' ) validation_message_summary_pass = generate_message_summary( merged_records, 'validation', verification_result_filter='pass' ) verification_message_summary_fail = generate_message_summary( merged_records, 'verification', verification_result_filter='fail' ) verification_message_summary_warn = generate_message_summary( merged_records, 'verification', verification_result_filter='warn' ) verification_message_summary_pass = generate_message_summary( merged_records, 'verification', verification_result_filter='pass' ) # Enrich message summaries with full record data for dummy report modal def enrich_message_summary_with_records(message_summary, all_records, message_type): """Add full record data to message summary for modal display.""" for item in message_summary: item['records'] = [] item['records_json'] = [] # JSON-serializable version for record_id in item.get('record_ids', []): for record in all_records: if record.get('occurrenceID') == record_id: item['records'].append(record) # Create JSON-serializable version json_record = { 'scientificName': record.get('scientificName', ''), 'eventDate': record.get('eventDate', ''), 'decimalLatitude': record.get('decimalLatitude', ''), 'decimalLongitude': record.get('decimalLongitude', ''), 'occurrenceID': record.get('occurrenceID', ''), 'validation_result': record.get('validation_result', ''), 'verification_result': record.get('verification_result', ''), } item['records_json'].append(json_record) break return message_summary # Enrich summaries for dummy report validation_message_summary_fail = enrich_message_summary_with_records( validation_message_summary_fail, merged_records, 'validation' ) validation_message_summary_warn = enrich_message_summary_with_records( validation_message_summary_warn, merged_records, 'validation' ) verification_message_summary_fail = enrich_message_summary_with_records( verification_message_summary_fail, merged_records, 'verification' ) verification_message_summary_warn = enrich_message_summary_with_records( verification_message_summary_warn, merged_records, 'verification' ) # Filter records to only show fail and warn on screen (including not_verified) fail_warn_records = [ record for record in merged_records if record.get('verification_result') in ['fail', 'warn', 'not_verified'] ] # Group records by category for display # Group records by category for display (same logic as generate_report) validation_fail_records = [ record for record in fail_warn_records if record.get('validation_result') == 'fail' ] # Validation warn records (only if validation didn't fail) validation_warn_records = [ record for record in fail_warn_records if record.get('validation_result') == 'warn' ] # Rules check fail records (only if validation passed) verification_fail_records = [ record for record in fail_warn_records if record.get('verification_result') == 'fail' and record.get('validation_result') == 'pass' ] # Rules check warn records (only if validation passed and verification didn't fail) verification_warn_records = [ record for record in fail_warn_records if record.get('verification_result') == 'warn' and record.get('validation_result') == 'pass' ] # Store in session for CSV download and debugging request.session['report_data'] = merged_records request.session['original_records'] = merged_records # For dummy data, use same records # Create dummy Record Cleaner JSON format dummy_rc_json = [] for i, record in enumerate(merged_records, start=1): dummy_rc_json.append({ 'id': i, 'name': record['scientificName'], 'date': record['eventDate'], 'sref': record.get('grid_reference', f"{record['decimalLatitude']},{record['decimalLongitude']}"), 'stage': record.get('lifeStage', '') }) request.session['record_cleaner_json'] = dummy_rc_json # Generate records warning for dummy data (25 records processed, assume 30 were fetched and 5 filtered out) dummy_original_count = 30 dummy_filtered_out_count = 5 dummy_processed_count = num_records records_warning = _generate_records_warning( dummy_original_count, dummy_filtered_out_count, dummy_processed_count, settings.RECORD_CLEANER_MAX_RECORDS ) # Prepare context context = { 'occurrences_count': num_records, 'summary': summary, 'validation_message_summary_fail': validation_message_summary_fail, 'validation_message_summary_warn': validation_message_summary_warn, 'validation_message_summary_pass': validation_message_summary_pass, 'verification_message_summary_fail': verification_message_summary_fail, 'verification_message_summary_warn': verification_message_summary_warn, 'verification_message_summary_pass': verification_message_summary_pass, 'merged_records': fail_warn_records, # Only fail/warn records for display 'validation_fail_records': validation_fail_records, 'validation_warn_records': validation_warn_records, 'verification_fail_records': verification_fail_records, 'verification_warn_records': verification_warn_records, 'input_type': 'dataset', 'data_resource_uid': 'DUMMY_DATA_RESOURCE', 'dataset_name': 'Dummy Test Dataset', 'search_url': None, 'records_warning': records_warning, 'is_dummy_report': True, # Flag to enable modal functionality } return render(request, 'atlas_record_cleaner_report/results.html', context)