Back to the dashboard · This notebook was executed on the n500_surveycto batch when the site was built. Synthetic data only.

1. Executive System Audit Overview

Comprehensive audit of survey paradata anomaly detection system.

Includes:

• Data validation
• Feature recomputation
• Rule engine verification
• Adversarial robustness testing
• Classification consistency
• Advanced correlation and fatigue analysis

2. Environment Setup

Import dependencies, resolve the project root directory, and load the live detector/reporting helpers used by the production pipeline.

import os, sys, glob, subprocess, warnings
from datetime import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import display

sns.set_theme(style='whitegrid')
warnings.filterwarnings('ignore', category=FutureWarning)
warnings.filterwarnings('ignore', category=FutureWarning, module='seaborn')

def resolve_root():
    current = os.getcwd()
    if os.path.exists(os.path.join(current, 'src')):
        return current
    elif os.path.exists(os.path.join(os.path.dirname(current), 'src')):
        return os.path.dirname(current)
    return current

PROJECT_ROOT = resolve_root()
os.chdir(PROJECT_ROOT)
if PROJECT_ROOT not in sys.path:
    sys.path.insert(0, PROJECT_ROOT)

from src.validation.anomaly_detector import AnomalyDetector
from src.validation.reporting import feature_dataset_view

print("Project Root:", PROJECT_ROOT)

Project Root: /tmp/tmpyydldua1

3. Data Ingestion

Load a selected combined feature dataset from the output directory, then attach the matching question-level and section-level tables when available.

PREFERRED_RUN_LABEL = os.environ.get('SKACE_NOTEBOOK_RUN_LABEL', 'canonical_validation')
ALLOW_FALLBACK_TO_LATEST = False

print("Preferred run label:", PREFERRED_RUN_LABEL)
print("Fallback to latest if preferred run is missing:", ALLOW_FALLBACK_TO_LATEST)

Preferred run label: n500_surveycto
Fallback to latest if preferred run is missing: False

def list_pipeline_outputs():
    output_dir = os.path.join(PROJECT_ROOT, 'output')
    pattern = os.path.join(output_dir, 'pipeline_output_*.csv')
    files = glob.glob(pattern)
    return sorted(files, key=os.path.getctime, reverse=True)

def find_dataset_for_label(run_label):
    if not run_label:
        return None, None
    candidate = os.path.join(PROJECT_ROOT, 'output', f'pipeline_output_{run_label}.csv')
    if os.path.exists(candidate):
        return candidate, pd.read_csv(candidate)
    return None, None

def find_dataset_with_fallback(preferred_run_label):
    output_dir = os.path.join(PROJECT_ROOT, 'output')
    selected_path, selected_df = find_dataset_for_label(preferred_run_label)
    if selected_path is not None:
        return selected_path, selected_df

    canonical_path, canonical_df = find_dataset_for_label('canonical_validation')
    if canonical_path is not None:
        return canonical_path, canonical_df

    bootstrap_path, bootstrap_df = find_dataset_for_label('notebook_bootstrap')
    if bootstrap_path is not None:
        return bootstrap_path, bootstrap_df

    if ALLOW_FALLBACK_TO_LATEST:
        available = list_pipeline_outputs()
        if available:
            latest = available[0]
            return latest, pd.read_csv(latest)

    # Historical compatibility fallbacks
    fallback = os.path.join(output_dir, 'final_combined_features.csv')
    if os.path.exists(fallback):
        return fallback, pd.read_csv(fallback)
        
    # Priority 3: Latest individual feature export
    pattern = os.path.join(output_dir, 'features_output_*.csv')
    files = glob.glob(pattern)
    if files:
        latest = max(files, key=os.path.getctime)
        return latest, pd.read_csv(latest)
        
    # Priority 4: Academic fallback
    academic_fallback = os.path.join(output_dir, 'final_combined_features_400series.csv')
    if os.path.exists(academic_fallback):
        return academic_fallback, pd.read_csv(academic_fallback)

    return None, None

def bootstrap_pipeline_output():
    canonical_inputs = [
        os.path.join(PROJECT_ROOT, 'data', 'odk.csv'),
        os.path.join(PROJECT_ROOT, 'data', 'surveycto.csv'),
    ]
    if not all(os.path.exists(path) for path in canonical_inputs):
        raise FileNotFoundError(
            "No feature files found in 'output/' and canonical data inputs are missing. "
            "Run `python run_pipeline.py --inputs ...` first."
        )

    cmd = [
        sys.executable,
        'run_pipeline.py',
        '--parser', 'auto',
        '--output-dir', 'output',
        '--label', 'notebook_bootstrap',
        '--inputs',
        *canonical_inputs,
    ]
    ground_truth = os.path.join(PROJECT_ROOT, 'data', 'ground_truth_labels.csv')
    if os.path.exists(ground_truth):
        cmd.extend(['--ground-truth', ground_truth])

    print("No pipeline output found. Bootstrapping canonical pipeline run...")
    result = subprocess.run(cmd, cwd=PROJECT_ROOT)
    if result.returncode != 0:
        raise RuntimeError(
            "Automatic notebook bootstrap failed. "
            "Run `python run_pipeline.py --inputs data/odk.csv data/surveycto.csv` manually."
        )

def load_selected_dataset():
    loaded_path, df = find_dataset_with_fallback(PREFERRED_RUN_LABEL)
    if loaded_path is None:
        available_outputs = list_pipeline_outputs()
        if available_outputs and not ALLOW_FALLBACK_TO_LATEST:
            raise FileNotFoundError(
                "Preferred run label was not found and fallback-to-latest is disabled. "
                "Set SKACE_NOTEBOOK_RUN_LABEL to one of the available pipeline_output labels "
                "or set ALLOW_FALLBACK_TO_LATEST = True."
            )
        bootstrap_pipeline_output()
        loaded_path, df = find_dataset_with_fallback(PREFERRED_RUN_LABEL)

    if loaded_path is None:
        raise FileNotFoundError("No feature files found in 'output/' directory after notebook bootstrap.")

    return loaded_path, df

def derive_run_label(path):
    basename = os.path.splitext(os.path.basename(path))[0]
    for prefix in ['pipeline_output_', 'feature_dataset_', 'features_output_']:
        if basename.startswith(prefix):
            return basename[len(prefix):]
    return None

def resolve_input_paths_from_loaded_df(source_df):
    if 'file_name' not in source_df.columns:
        return []

    resolved = []
    for file_name in source_df['file_name'].dropna().astype(str).unique():
        matches = glob.glob(os.path.join(PROJECT_ROOT, 'data', '**', file_name), recursive=True)
        if not matches:
            return []
        resolved.append(matches[0])
    return resolved

def load_granular_outputs(source_path):
    run_label = derive_run_label(source_path)
    question_path = None
    section_path = None
    question_df = pd.DataFrame()
    section_df = pd.DataFrame()

    if run_label:
        question_candidate = os.path.join(PROJECT_ROOT, 'output', f'question_features_{run_label}.csv')
        section_candidate = os.path.join(PROJECT_ROOT, 'output', f'section_features_{run_label}.csv')

        if os.path.exists(question_candidate):
            question_path = question_candidate
            question_df = pd.read_csv(question_candidate)

        if os.path.exists(section_candidate):
            section_path = section_candidate
            section_df = pd.read_csv(section_candidate)

    return question_path, question_df, section_path, section_df

def ensure_granular_outputs(source_path, source_df):
    question_path, question_df, section_path, section_df = load_granular_outputs(source_path)
    if not question_df.empty and not section_df.empty:
        return question_path, question_df, section_path, section_df

    run_label = derive_run_label(source_path)
    input_paths = resolve_input_paths_from_loaded_df(source_df)
    if not run_label or not input_paths:
        return question_path, question_df, section_path, section_df

    cmd = [
        sys.executable,
        'run_pipeline.py',
        '--parser', 'auto',
        '--output-dir', 'output',
        '--label', run_label,
        '--inputs',
        *input_paths,
    ]
    ground_truth = os.path.join(PROJECT_ROOT, 'data', 'ground_truth_labels.csv')
    if os.path.exists(ground_truth) and run_label in ['canonical_validation', 'notebook_bootstrap']:
        cmd.extend(['--ground-truth', ground_truth])

    print("Granular outputs missing for selected run. Rebuilding them from raw inputs...")
    result = subprocess.run(cmd, cwd=PROJECT_ROOT)
    if result.returncode != 0:
        print("Granular output rebuild failed; continuing without them.")
        return question_path, question_df, section_path, section_df

    return load_granular_outputs(source_path)

available_outputs = list_pipeline_outputs()
print("Available pipeline outputs (newest first):")
for path in available_outputs[:10]:
    print(" -", os.path.basename(path))

loaded_path, df = load_selected_dataset()
question_path, question_df, section_path, section_df = ensure_granular_outputs(loaded_path, df)
CURRENT_RUN_LABEL = derive_run_label(loaded_path) or 'manual'
print("Data source:", os.path.abspath(loaded_path))
print("Shape:", df.shape)
print("Columns:", len(df.columns))
print("Resolved run label:", CURRENT_RUN_LABEL)
print("Question feature file:", question_path if question_path else "not found for this run")
print("Section feature file:", section_path if section_path else "not found for this run")

Available pipeline outputs (newest first):
 - pipeline_output_n500_surveycto.csv
Data source: /tmp/tmpyydldua1/output/pipeline_output_n500_surveycto.csv
Shape: (500, 72)
Columns: 72
Resolved run label: n500_surveycto
Question feature file: /tmp/tmpyydldua1/output/question_features_n500_surveycto.csv
Section feature file: /tmp/tmpyydldua1/output/section_features_n500_surveycto.csv

3.1 Granular Feature Tables

Preview the question-level and section-level outputs emitted by the pipeline for the same run label. These tables preserve the detailed feature grain before the interview-level feature dataset is flattened for anomaly detection.

if question_df.empty:
    print("Question-level output not available for the loaded run.")
else:
    print("Question-level shape:", question_df.shape)
    display(question_df.head())

if section_df.empty:
    print("Section-level output not available for the loaded run.")
else:
    print("Section-level shape:", section_df.shape)
    display(section_df.head())

Question-level shape: (10000, 13)

	file_name	interview_id	field_name	Q-01_response_time_ms	Q-02_time_to_first_interaction	Q-03_edit_count	Q-04_revisit_flag	Q-05_revisit_count	Q-06_field_completion_ratio	Q-07_idle_time_ms	Q-08_keystroke_count	Q-09_constraint_trigger_count	Q-10_first_to_last_interaction_duration
0	audit_logs_surveycto.csv	INT-00001	demographics_q01	5881.0	5881.0	0	0	0	1.0	0	1	0	5881
1	audit_logs_surveycto.csv	INT-00001	demographics_q02	6118.0	6118.0	0	0	0	1.0	0	1	0	6118
2	audit_logs_surveycto.csv	INT-00001	demographics_q03	5683.0	5683.0	1	0	0	1.0	494	2	0	5683
3	audit_logs_surveycto.csv	INT-00001	demographics_q04	6295.0	6295.0	0	0	0	1.0	0	1	0	6295
4	audit_logs_surveycto.csv	INT-00001	economic_q01	5827.0	5827.0	0	0	0	1.0	0	1	0	5827

Section-level shape: (2500, 13)

	file_name	interview_id	section_name	S-01_mean_section_pace	S-02_section_duration_ms	S-03_within_section_variance	S-05_pause_count_5s	S-06_pause_count_30s	S-07_mean_pause_duration_ms	S-08_max_pause_duration_ms	S-09_back_nav_count_section	S-10_section_completion_rate	S-04_section_to_section_pace_ratio
0	audit_logs_surveycto.csv	INT-00001	demographics	5994.25	26770	71822.250000	0	0	623.25	1160	0	1.0	0.993050
1	audit_logs_surveycto.csv	INT-00001	economic	5960.50	27210	10441.666667	0	0	553.60	948	0	1.0	0.987459
2	audit_logs_surveycto.csv	INT-00001	health	6089.75	27633	6765.583333	0	0	534.80	870	0	1.0	1.008871
3	audit_logs_surveycto.csv	INT-00001	household	6072.25	28578	17492.916667	0	0	737.80	1102	0	1.0	1.005972
4	audit_logs_surveycto.csv	INT-00001	nutrition	6064.25	27458	52108.250000	0	0	725.25	1060	0	1.0	1.004647

4. Data Validation Layer

Apply general audit diagnostics: missing data, negative durations, invalid ratios, and logical inconsistencies between dependent metrics. These audit_* checks are for exploratory auditing and are separate from the production detector.

# Convert object columns to numeric where possible (pandas 3.x compatible)
for col in df.select_dtypes(include='object').columns:
    converted = pd.to_numeric(df[col], errors='coerce')
    if converted.notna().any():
        df.loc[:, col] = converted

# Missing + invalid checks
df.loc[:, 'audit_missing_data'] = df.isna().any(axis=1)

if 'V-01_total_interview_duration_ms' in df.columns:
    df.loc[:, 'audit_negative_duration'] = df['V-01_total_interview_duration_ms'] < 0

if 'V-04_speed_trajectory_ratio' in df.columns:
    df.loc[:, 'audit_invalid_ratio'] = df['V-04_speed_trajectory_ratio'].isna() | (df['V-04_speed_trajectory_ratio'] < 0)

# Logical inconsistencies
if all(c in df.columns for c in ['T-06_rt_mean', 'T-09_rt_min', 'T-10_rt_max']):
    df.loc[:, 'audit_inconsistent_metrics'] = (
        (df['T-06_rt_mean'] < df['T-09_rt_min']) |
        (df['T-06_rt_mean'] > df['T-10_rt_max'])
    )

print("Audit diagnostics created")

Audit diagnostics created

/tmp/ipykernel_30/1992238569.py:2: Pandas4Warning: For backward compatibility, 'str' dtypes are included by select_dtypes when 'object' dtype is specified. This behavior is deprecated and will be removed in a future version. Explicitly pass 'str' to `include` to select them, or to `exclude` to remove them and silence this warning.
See https://pandas.pydata.org/docs/user_guide/migration-3-strings.html#string-migration-select-dtypes for details on how to write code that works with pandas 2 and 3.
  for col in df.select_dtypes(include='object').columns:

5. Feature Recalculation

Recompute only the aggregated metrics that can be verified from the flattened feature table itself.

Some raw-log metrics, such as back-navigation rate, cannot be recomputed exactly from the final one-row-per-interview dataset because the event sequence is no longer present.

if all(c in df.columns for c in ['T-06_rt_mean', 'T-07_rt_median', 'T-09_rt_min', 'T-10_rt_max']):
    df.loc[:, 'flag_stat_order_issue'] = (
        (df['T-09_rt_min'] > df['T-06_rt_mean']) |
        (df['T-06_rt_mean'] > df['T-10_rt_max']) |
        (df['T-09_rt_min'] > df['T-07_rt_median']) |
        (df['T-07_rt_median'] > df['T-10_rt_max'])
    )
else:
    df.loc[:, 'flag_stat_order_issue'] = False

6. Live Detector Reproduction

Recompute anomaly flags, scores, categories, and status using the production AnomalyDetector directly. This avoids drift between the notebook and the live pipeline logic.

detector_input = feature_dataset_view(df)
detector = AnomalyDetector()
recomputed = detector.detect(detector_input)

for col in [
    'anomaly_score', 'anomaly_category', 'status', 'flagged_features',
    'iqr_flags', 'rule_flags', 'is_data_quality_issue', 'is_anomalous_behavior', 'is_anomalous'
]:
    if col in recomputed.columns:
        df.loc[:, f'recomputed_{col}'] = recomputed[col]

df.loc[:, 'computed_anomaly_score'] = df.get('recomputed_anomaly_score', pd.Series(0, index=df.index))
df.loc[:, 'computed_category'] = df.get('recomputed_anomaly_category', pd.Series('unknown', index=df.index))

if pd.to_numeric(df['computed_anomaly_score'], errors='coerce').max() > 0:
    df.loc[:, 'normalized_score'] = pd.to_numeric(df['computed_anomaly_score'], errors='coerce') / pd.to_numeric(df['computed_anomaly_score'], errors='coerce').max()
else:
    df.loc[:, 'normalized_score'] = 0.0

print("Production detector reapplied to", len(recomputed), "rows")

Production detector reapplied to 500 rows

7. Audit Consistency Checks

Compare the live detector recomputation against the source file when the loaded dataset already contains pipeline-produced anomaly outputs.

comparison_pairs = [
    ('anomaly_score', 'recomputed_anomaly_score'),
    ('anomaly_category', 'recomputed_anomaly_category'),
    ('status', 'recomputed_status'),
    ('flagged_features', 'recomputed_flagged_features'),
]

comparison_summary = []
for source_col, recomputed_col in comparison_pairs:
    if source_col in df.columns and recomputed_col in df.columns:
        mismatches = (df[source_col].fillna('').astype(str) != df[recomputed_col].fillna('').astype(str)).sum()
        comparison_summary.append({
            'column': source_col,
            'mismatches': int(mismatches),
        })

if comparison_summary:
    comparison_df = pd.DataFrame(comparison_summary)
    display(comparison_df)
else:
    print("Loaded dataset does not contain source anomaly outputs to compare against.")

	column	mismatches
0	anomaly_score	0
1	anomaly_category	0
2	status	0
3	flagged_features	0

8. Summary Statistics

Descriptive statistics for all numeric features and flag distribution counts.

print(df.describe())

print("\nFlag Distribution:")
flags = [col for col in df.columns if col.startswith('flag_') or col.startswith('audit_')]
for f in flags:
    print(f, df[f].sum())

       V-01_total_interview_duration_ms  V-02_speed_trajectory_early  \
count                        500.000000                   500.000000   
mean                      159774.252000                  6373.288660   
std                        58342.008289                  2407.675857   
min                        11023.000000                   560.666667   
25%                       153335.500000                  5756.916667   
50%                       175840.000000                  6896.950000   
75%                       194116.750000                  7805.416667   
max                       263908.000000                 12397.428571   

       V-03_speed_trajectory_late  V-04_speed_trajectory_ratio  \
count                  500.000000                   500.000000   
mean                  7711.235543                     1.217096   
std                   3088.785413                     0.373439   
min                    524.500000                     0.532564   
25%                   6952.428571                     0.952463   
50%                   8168.333333                     1.157913   
75%                   9644.291667                     1.413416   
max                  16125.166667                     2.569398   

       V-06_back_navigation_total_count  V-05_back_navigation_rate  \
count                        500.000000                 500.000000   
mean                           2.086000                   0.093001   
std                            2.119336                   0.095735   
min                            0.000000                   0.000000   
25%                            1.000000                   0.041250   
50%                            2.000000                   0.086957   
75%                            3.000000                   0.131917   
max                           10.000000                   0.454545   

       V-07_time_of_day_hour  V-08_day_of_week  V-09_session_length_vs_median  \
count             500.000000        500.000000                     500.000000   
mean               11.728000          1.560000                       0.908634   
std                 6.578493          1.047183                       0.331790   
min                 0.000000          0.000000                       0.062688   
25%                 6.000000          1.000000                       0.872017   
50%                12.000000          2.000000                       1.000000   
75%                17.000000          2.000000                       1.103940   
max                23.000000          3.000000                       1.500842   

       V-10_zero_back_nav_flag  ...  is_anomalous_behavior  is_anomalous  \
count               500.000000  ...             500.000000    500.000000   
mean                  0.284000  ...               0.200000      0.200000   
std                   0.451388  ...               0.400401      0.400401   
min                   0.000000  ...               0.000000      0.000000   
25%                   0.000000  ...               0.000000      0.000000   
50%                   0.000000  ...               0.000000      0.000000   
75%                   1.000000  ...               0.000000      0.000000   
max                   1.000000  ...               1.000000      1.000000   

       recomputed_anomaly_score  recomputed_iqr_flags  recomputed_rule_flags  \
count                     500.0                 500.0                  500.0   
mean                      1.418                 0.224                  1.194   
std                    2.798381               0.41734               2.410806   
min                         0.0                   0.0                    0.0   
25%                         0.0                   0.0                    0.0   
50%                         0.0                   0.0                    0.0   
75%                         0.0                   0.0                    0.0   
max                         8.0                   1.0                    7.0   

       recomputed_is_data_quality_issue  recomputed_is_anomalous_behavior  \
count                             500.0                        500.000000   
mean                                0.0                          0.200000   
std                                 0.0                          0.400401   
min                                 0.0                          0.000000   
25%                                 0.0                          0.000000   
50%                                 0.0                          0.000000   
75%                                 0.0                          0.000000   
max                                 0.0                          1.000000   

       recomputed_is_anomalous  computed_anomaly_score  normalized_score  
count               500.000000                   500.0             500.0  
mean                  0.200000                   1.418           0.17725  
std                   0.400401                2.798381          0.349798  
min                   0.000000                     0.0               0.0  
25%                   0.000000                     0.0               0.0  
50%                   0.000000                     0.0               0.0  
75%                   0.000000                     0.0               0.0  
max                   1.000000                     8.0               1.0  

[8 rows x 75 columns]

Flag Distribution:
flag_invalid_ratio 0
flag_missing_data 0
flag_inconsistent_metrics 0
flag_too_fast 0
flag_too_slow 0
flag_short_relative_duration 66
flag_low_mean_rt 66
flag_zero_back_nav 66
flag_low_entropy 66
flag_uniform_timing 67
flag_low_rt_iqr 100
flag_low_section_variance 100
flag_high_back_nav 0
flag_moderate_back_nav 37
flag_long_pause_relative 29
flag_elevated_pause 0
flag_rule_pause 0
flag_iqr_duration 69
flag_iqr_rt_mean 69
flag_iqr_entropy 100
flag_iqr_variance 11
audit_missing_data 385
audit_negative_duration 0
audit_invalid_ratio 0
audit_inconsistent_metrics 0
flag_stat_order_issue 0

9. Visualization

Distribution of anomaly scores and category breakdown across the dataset.

plt.figure()
if df['computed_anomaly_score'].nunique() > 1:
    sns.histplot(df['computed_anomaly_score'], bins=20, kde=True)
else:
    sns.histplot(df['computed_anomaly_score'], bins=20, kde=False)
plt.title('Anomaly Score Distribution')
plt.show()

plt.figure()
sns.countplot(
    data=df,
    x='computed_category',
    hue='computed_category',
    order=['normal', 'suspicious', 'anomalous', 'dq_issue'],
    hue_order=['normal', 'suspicious', 'anomalous', 'dq_issue'],
    palette={'normal':'#2ca02c', 'suspicious':'#ff7f0e', 'anomalous':'#d62728', 'dq_issue':'#9467bd'},
    legend=False
)
plt.title('Category Distribution')
plt.show()

10. Adversarial Edge Case Detection

Identify interviews with extreme or degenerate data: missing values, non-positive durations, zero entropy.

edge_condition = df.get('audit_missing_data', pd.Series(False, index=df.index)) == True

if 'V-01_total_interview_duration_ms' in df.columns:
    edge_condition = edge_condition | (df['V-01_total_interview_duration_ms'] <= 0)

if 'T-03_rt_entropy' in df.columns:
    edge_condition = edge_condition | (df['T-03_rt_entropy'] == 0)

edge_cases = df[edge_condition]
print("Edge Cases Detected:", len(edge_cases))
display(edge_cases.head() if not edge_cases.empty else "No edge cases matched.")

Edge Cases Detected: 445

	file_name	interview_id	V-01_total_interview_duration_ms	V-02_speed_trajectory_early	V-03_speed_trajectory_late	V-04_speed_trajectory_ratio	V-06_back_navigation_total_count	V-05_back_navigation_rate	V-07_time_of_day_hour	V-08_day_of_week	...	recomputed_status	recomputed_flagged_features	recomputed_iqr_flags	recomputed_rule_flags	recomputed_is_data_quality_issue	recomputed_is_anomalous_behavior	recomputed_is_anomalous	computed_anomaly_score	computed_category	normalized_score
1	audit_logs_surveycto.csv	INT-00002	177628	5294.500000	8616.857143	1.627511	0	0.000000	8	0	...	CLEAN		0	0	0	0	0	0	normal	0.0
3	audit_logs_surveycto.csv	INT-00004	202789	8153.166667	13090.600000	1.605585	4	0.173913	8	0	...	CLEAN		0	0	0	0	0	0	normal	0.0
4	audit_logs_surveycto.csv	INT-00005	186498	7015.142857	9919.666667	1.414036	2	0.090909	8	0	...	CLEAN		0	0	0	0	0	0	normal	0.0
5	audit_logs_surveycto.csv	INT-00006	179285	5892.285714	6152.666667	1.044190	2	0.086957	8	0	...	CLEAN		0	0	0	0	0	0	normal	0.0
6	audit_logs_surveycto.csv	INT-00007	186137	7003.285714	8223.800000	1.174277	1	0.041667	9	0	...	CLEAN		0	0	0	0	0	0	normal	0.0

5 rows × 89 columns

11. Final Output

Export the audit-enriched dataset with all recomputed flags, scores, and categories.

timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
output_path = os.path.join(PROJECT_ROOT, 'output', f'audit_results_{CURRENT_RUN_LABEL}_{timestamp}.csv')
os.makedirs(os.path.dirname(output_path), exist_ok=True)
df.to_csv(output_path, index=False)
print("Audit results saved to:", output_path)

Audit results saved to: /tmp/tmpyydldua1/output/audit_results_n500_surveycto_20261018_103439.csv

12. Correlation Analysis

Correlation heatmap of key structural and statistical paradata features to identify collinear metrics.

features_to_correlate = [
    'computed_anomaly_score', 'V-01_total_interview_duration_ms',
    'V-05_back_navigation_rate', 'T-01_rt_variance', 'T-03_rt_entropy',
    'T-06_rt_mean', 'S-05_pause_count_5s', 'S-10_section_completion_rate'
]
available_features = [f for f in features_to_correlate if f in df.columns]

if available_features:
    corr = df[available_features].corr()
    plt.figure(figsize=(10, 8))
    sns.heatmap(corr, annot=True, cmap='coolwarm', fmt=".2f", vmin=-1, vmax=1)
    plt.title('Feature Correlation Heatmap')
    plt.tight_layout()
    plt.show()
else:
    print("Required features for correlation analysis are missing from the dataframe.")

13. Anomaly vs. Session Duration

Examining the relationship between total survey duration and the calculated anomaly score.

if 'V-01_total_interview_duration_ms' in df.columns and 'computed_anomaly_score' in df.columns:
    plt.figure(figsize=(10, 6))
    
    # Optional: filter out negative durations or highly extreme outliers for a cleaner plot
    plot_df = df[df['V-01_total_interview_duration_ms'] > 0]
    
    if not plot_df.empty:
        sns.scatterplot(
            data=plot_df, 
            x='V-01_total_interview_duration_ms', 
            y='computed_anomaly_score',
            hue='computed_category',
            palette={'normal':'#2ca02c', 'suspicious':'#ff7f0e', 'anomalous':'#d62728', 'dq_issue':'#9467bd'},
            alpha=0.7
        )
        plt.title('Anomaly Score vs Total Interview Duration')
        plt.xlabel('Total Interview Duration (ms)')
        plt.ylabel('Computed Anomaly Score')
        plt.xscale('log') # Log scale is often better for duration
        plt.tight_layout()
        plt.show()
    else:
        print("No valid duration data to plot.")

14. Section-Level Fatigue Analysis

Analyzing section-level features (pauses and section completion) by anomaly category to discover where anomalous respondents start showing fatigue.

s_features = ['S-05_pause_count_5s', 'S-07_mean_pause_duration_ms', 'S-10_section_completion_rate']
available_s = [f for f in s_features if f in df.columns]

if available_s and 'computed_category' in df.columns:
    fig, axes = plt.subplots(1, len(available_s), figsize=(6 * len(available_s), 5))
    if len(available_s) == 1:
        axes = [axes]
    
    for ax, feat in zip(axes, available_s):
        sns.boxplot(
            data=df, 
            x='computed_category', 
            y=feat, 
            hue='computed_category',
            ax=ax, 
            order=['normal', 'suspicious', 'anomalous', 'dq_issue'],
            hue_order=['normal', 'suspicious', 'anomalous', 'dq_issue'],
            palette={'normal':'#2ca02c', 'suspicious':'#ff7f0e', 'anomalous':'#d62728', 'dq_issue':'#9467bd'},
            dodge=False,
            legend=False
        )
        ax.set_title(f'{feat} by Category')
        ax.tick_params(axis='x', rotation=45)
    
    plt.tight_layout()
    plt.show()
else:
    print("S-Series parameters not available for Section-Level Fatigue Analysis.")

15. Multidimensional Feature Pairplot

Pairwise scatterplots to visually cluster behavioral subsets using timing and structural metrics.

pairplot_features = ['T-03_rt_entropy', 'T-06_rt_mean', 'V-05_back_navigation_rate', 'computed_category']
available_pair = [f for f in pairplot_features if f in df.columns]

if len(available_pair) > 1 and 'computed_category' in available_pair:
    sns.pairplot(
        df[available_pair], 
        hue='computed_category', 
        palette={'normal':'#2ca02c', 'suspicious':'#ff7f0e', 'anomalous':'#d62728', 'dq_issue':'#9467bd'}, 
        corner=True,
        diag_kind='hist',
        plot_kws={'alpha': 0.6}
    )
    plt.suptitle('Pairplot of Key Paradata Metrics', y=1.02)
    plt.show()
else:
    print("Required features for pairplot are missing.")