fix: Universal gyro activity filtering to skip ground test logs

OVERVIEW.md

@@ -118,9 +118,12 @@ src/
 
 ### **Smart Export Filtering**
 - **Duration-based:** < 5s skipped, 5–15s exported only if data density > 1500 fps, > 15s exported
-- **Gyro activity detection:** Minimal gyro variance indicates ground test vs. actual flight
+- **Gyro activity detection:** Minimal gyro range (< 500) indicates ground test vs. actual flight
+- **Thresholds:**
+  - `FALLBACK_MIN_FRAMES = 7_500` (~5 seconds at 1500fps)
+  - `MIN_GYRO_RANGE = 500.0` (actual flights >500, ground tests <500)
 - **Configurable:** Available via library API `should_skip_export()` and `has_minimal_gyro_activity()` for programmatic control
-- **Override:** `--force-export` flag or `force_export` option bypasses filtering heuristics
+- **Override:** `--force-export` flag (CLI) or `force_export` option (library) bypasses all filtering heuristics
 
 ### **Library API**
 - **Complete Data Access:** Programmatic access to all BBL data structures

README.md

@@ -64,8 +64,11 @@ To reduce noise from test arm/disarm logs:
 - < 5s: skipped
 - 5–15s: exported only if data density > 1500 fps
 - > 15s: exported
+- Minimal gyro activity: skipped (ground test detection)
 
-Use `--force-export` to export everything.
+Gyro range threshold: 500 (below = likely ground test, above = potential flight)
+
+Use `--force-export` to export all logs regardless of filtering criteria.
 
 ## Documentation
 

src/filters.rs

@@ -30,7 +30,7 @@ pub fn should_skip_export(log: &BBLLog, force_export: bool) -> (bool, String) {
     const VERY_SHORT_DURATION_MS: u64 = 5_000; // 5 seconds - always skip
     const SHORT_DURATION_MS: u64 = 15_000; // 15 seconds - threshold for normal logs
     const MIN_DATA_DENSITY_FPS: f64 = 1500.0; // Minimum fps for short logs
-    const FALLBACK_MIN_FRAMES: u32 = 7_500; // ~5 seconds at 1500 fps (fallback when no duration)
+    const FALLBACK_MIN_FRAMES: u32 = 7_500; // ~5 seconds at 1500 fps, ~1 second at 8000 fps
 
     // Check if we have duration information
     let duration_us = log.duration_us();
@@ -67,24 +67,22 @@ pub fn should_skip_export(log: &BBLLog, force_export: bool) -> (bool, String) {
         }
 
         // Normal logs: > 15 seconds → Check for minimal gyro activity (ground tests)
-        if duration_ms >= SHORT_DURATION_MS {
-            let (is_minimal_movement, max_variance) = has_minimal_gyro_activity(log);
-            if is_minimal_movement {
-                return (
-                    true,
-                    format!(
-                        "minimal gyro activity ({:.1} variance) - likely ground test",
-                        max_variance
-                    ),
-                );
-            }
+        let (is_minimal_movement, max_range) = has_minimal_gyro_activity(log);
+        if is_minimal_movement {
+            return (
+                true,
+                format!(
+                    "minimal gyro activity ({:.1} range) - likely ground test",
+                    max_range
+                ),
+            );
         }
 
         return (false, String::new());
     }
 
-    // No duration information available, fall back to frame count
-    // Skip if very low frame count (equivalent to <5s at minimum viable fps)
+    // No duration information available, fall back to frame count and gyro variance
+    // Skip if very low frame count (equivalent to <10s at minimum viable fps)
     if log.stats.total_frames < FALLBACK_MIN_FRAMES {
         return (
             true,
@@ -95,23 +93,41 @@ pub fn should_skip_export(log: &BBLLog, force_export: bool) -> (bool, String) {
         );
     }
 
-    // Sufficient frames without duration info, keep it
+    // For logs without duration but sufficient frames, apply gyro range check
+    // This catches INAV logs and older Betaflight logs that lack duration info
+    let (is_minimal_movement, max_range) = has_minimal_gyro_activity(log);
+    if is_minimal_movement {
+        return (
+            true,
+            format!(
+                "minimal gyro activity ({:.1} range) - likely ground test (no duration info)",
+                max_range
+            ),
+        );
+    }
+
+    // Sufficient frames and meaningful gyro activity, keep it
     (false, String::new())
 }
 
-/// Analyzes gyro variance to detect ground tests vs actual flight
+/// Analyzes gyro activity to detect ground tests vs actual flight
+///
+/// Uses the maximum axis range (max - min) across all three gyro axes to detect minimal movement.
+/// This approach is less scale-sensitive than variance-based methods, though results still depend
+/// on gyro sensor units and firmware scaling. Real flights typically show gyro ranges in the thousands,
+/// while ground tests show minimal variation (sensor noise only).
 ///
 /// Returns true if the log appears to be a static ground test (minimal movement)
 ///
 /// # Arguments
 /// * `log` - The BBL log to analyze
 ///
 /// # Returns
-/// Tuple of (is_minimal_movement, max_variance_value)
+/// Tuple of (is_minimal_movement, max_gyro_range)
 pub fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) {
     // Conservative thresholds to avoid false-skips
     const MIN_SAMPLES_FOR_ANALYSIS: usize = 15; // Reduced for limited sample data
-    const VERY_LOW_GYRO_VARIANCE_THRESHOLD: f64 = 0.3; // More aggressive threshold for ground test detection
+    const MIN_GYRO_RANGE: f64 = 500.0; // Minimum range to distinguish static bench tests from gentle flights
 
     let mut gyro_x_values = Vec::new();
     let mut gyro_y_values = Vec::new();
@@ -123,14 +139,14 @@ pub fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) {
         for (frame_type, frames) in debug_frames {
             if *frame_type == 'I' || *frame_type == 'P' {
                 for frame in frames {
-                    if let Some(gyro_x) = frame.data.get("gyroADC[0]") {
-                        if let Some(gyro_y) = frame.data.get("gyroADC[1]") {
-                            if let Some(gyro_z) = frame.data.get("gyroADC[2]") {
-                                gyro_x_values.push(*gyro_x as f64);
-                                gyro_y_values.push(*gyro_y as f64);
-                                gyro_z_values.push(*gyro_z as f64);
-                            }
-                        }
+                    if let (Some(&gx), Some(&gy), Some(&gz)) = (
+                        frame.data.get("gyroADC[0]"),
+                        frame.data.get("gyroADC[1]"),
+                        frame.data.get("gyroADC[2]"),
+                    ) {
+                        gyro_x_values.push(gx as f64);
+                        gyro_y_values.push(gy as f64);
+                        gyro_z_values.push(gz as f64);
                     }
                 }
             }
@@ -140,14 +156,14 @@ pub fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) {
     // Fallback to frames if debug_frames not available or insufficient data
     if gyro_x_values.len() < MIN_SAMPLES_FOR_ANALYSIS {
         for frame in &log.frames {
-            if let Some(gyro_x) = frame.data.get("gyroADC[0]") {
-                if let Some(gyro_y) = frame.data.get("gyroADC[1]") {
-                    if let Some(gyro_z) = frame.data.get("gyroADC[2]") {
-                        gyro_x_values.push(*gyro_x as f64);
-                        gyro_y_values.push(*gyro_y as f64);
-                        gyro_z_values.push(*gyro_z as f64);
-                    }
-                }
+            if let (Some(&gx), Some(&gy), Some(&gz)) = (
+                frame.data.get("gyroADC[0]"),
+                frame.data.get("gyroADC[1]"),
+                frame.data.get("gyroADC[2]"),
+            ) {
+                gyro_x_values.push(gx as f64);
+                gyro_y_values.push(gy as f64);
+                gyro_z_values.push(gz as f64);
             }
         }
     }
@@ -157,28 +173,67 @@ pub fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) {
         return (false, 0.0); // Not enough data, don't skip (conservative approach)
     }
 
-    // Calculate variance for each axis
-    let variance_x = calculate_variance(&gyro_x_values);
-    let variance_y = calculate_variance(&gyro_y_values);
-    let variance_z = calculate_variance(&gyro_z_values);
+    // Calculate range (max - min) for each axis
+    // Ground tests show minimal range due to sensor noise only, while flights show large excursions.
+    // Note: Results depend on gyro sensor units (varies by firmware version and sensor type)
+    let range_x = calculate_range(&gyro_x_values);
+    let range_y = calculate_range(&gyro_y_values);
+    let range_z = calculate_range(&gyro_z_values);
 
-    // Use the maximum variance across all axes
-    let max_variance = variance_x.max(variance_y).max(variance_z);
+    // Use the maximum range across all axes as the detection metric
+    let max_range = range_x.max(range_y).max(range_z);
 
-    // Very conservative: only skip if the highest variance across all axes is extremely low
-    // This means the aircraft was essentially stationary (ground test)
-    let is_minimal = max_variance < VERY_LOW_GYRO_VARIANCE_THRESHOLD;
+    // If maximum axis range is below threshold, classify as ground test
+    // Threshold of MIN_GYRO_RANGE (500.0) catches static bench tests while allowing gentle/beginner flights
+    // True ground tests: <500 (sensor noise), Gentle flights: >500 (real movement)
+    let is_minimal = max_range < MIN_GYRO_RANGE;
 
-    (is_minimal, max_variance)
+    (is_minimal, max_range)
+}
+
+/// Calculate range (max - min) of a dataset
+///
+/// Returns 0.0 for empty datasets. If input contains NaN values, the result will be NaN
+/// (conservative: won't trigger skip logic). This ensures data quality issues are caught
+/// rather than silently passing through.
+///
+/// # Arguments
+/// * `values` - Slice of f64 values to compute range for
+///
+/// # Returns
+/// The range of the dataset (max - min), or 0.0 if empty, or NaN if input contains NaN
+pub fn calculate_range(values: &[f64]) -> f64 {
+    if values.is_empty() {
+        return 0.0;
+    }
+
+    // Check for NaN values first (conservative: propagate NaN to catch data quality issues)
+    if values.iter().any(|v| v.is_nan()) {
+        return f64::NAN;
+    }
+
+    let min = values.iter().copied().fold(f64::INFINITY, f64::min);
+    let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max);
+
+    max - min
 }
 
 /// Calculate variance of a dataset
 ///
+/// # Deprecation Notice
+/// This function is no longer used by the filtering logic. The range-based detection approach
+/// (see [`calculate_range()`]) is now preferred as it reduces sensitivity to scale differences.
+///
 /// # Arguments
 /// * `values` - Slice of f64 values to compute variance for
 ///
 /// # Returns
 /// The variance of the dataset
+#[deprecated(
+    since = "1.0.0",
+    note = "Use calculate_range() instead. This function is kept for backward compatibility only."
+)]
+#[allow(dead_code)]
 pub fn calculate_variance(values: &[f64]) -> f64 {
     if values.len() < 2 {
         return 0.0;
@@ -259,8 +314,8 @@ mod tests {
 
     #[test]
     fn test_fallback_to_frame_count() {
-        // No duration info, but sufficient frame count should keep
-        let log = create_test_log(0, 0, 8000); // 8000 frames, no duration
+        // No duration info, but sufficient frame count should keep (above 7,500 threshold)
+        let log = create_test_log(0, 0, 16000); // 16000 frames, no duration
         let (should_skip, _) = should_skip_export(&log, false);
         assert!(
             !should_skip,
@@ -281,24 +336,101 @@ mod tests {
     }
 
     #[test]
-    fn test_calculate_variance() {
-        let values = vec![1.0, 2.0, 3.0, 4.0, 5.0];
-        let variance = calculate_variance(&values);
-        // Expected variance: mean=3, variance=2.0
-        assert!((variance - 2.0).abs() < 0.001);
+    fn test_no_duration_with_minimal_gyro_activity() {
+        // No duration info, sufficient frames, but minimal gyro range (ground test)
+        use crate::types::DecodedFrame;
+        use std::collections::HashMap;
+
+        let mut log = create_test_log(0, 0, 16000); // 16000 frames, no duration
+
+        // Create frames with minimal gyro variation (ground test pattern)
+        // Gyro range will be < MIN_GYRO_RANGE (500.0) — representing sensor noise only
+        for i in 0..100 {
+            let mut data = HashMap::new();
+            data.insert("gyroADC[0]".to_string(), 10 + (i % 5) as i32); // Range: 5
+            data.insert("gyroADC[1]".to_string(), -15 + (i % 7) as i32); // Range: 7
+            data.insert("gyroADC[2]".to_string(), 20 + (i % 10) as i32); // Range: 10
+
+            log.frames.push(DecodedFrame {
+                frame_type: 'P',
+                timestamp_us: i as u64 * 1000,
+                loop_iteration: i,
+                data,
+            });
+        }
+
+        let (should_skip, reason) = should_skip_export(&log, false);
+        assert!(
+            should_skip,
+            "Expected to skip ground test with minimal gyro activity"
+        );
+        assert!(
+            reason.contains("minimal gyro activity"),
+            "Expected 'minimal gyro activity' reason, got: {}",
+            reason
+        );
+    }
+
+    #[test]
+    fn test_no_duration_with_flight_gyro_activity() {
+        // No duration info, sufficient frames, high gyro range (actual flight)
+        use crate::types::DecodedFrame;
+        use std::collections::HashMap;
+
+        let mut log = create_test_log(0, 0, 16000); // 16000 frames, no duration
+
+        // Create frames with flight-typical gyro variation (large excursions)
+        // Gyro range will be > MIN_GYRO_RANGE (500.0) (actual flight movement)
+        for i in 0..100 {
+            let mut data = HashMap::new();
+            // Simulate flight with gyro values ranging -3000 to +3000
+            data.insert("gyroADC[0]".to_string(), -3000 + (i * 60) as i32); // Large range
+            data.insert("gyroADC[1]".to_string(), -2500 + (i * 50) as i32); // Large range
+            data.insert("gyroADC[2]".to_string(), -2000 + (i * 40) as i32); // Large range
+
+            log.frames.push(DecodedFrame {
+                frame_type: 'P',
+                timestamp_us: i as u64 * 1000,
+                loop_iteration: i,
+                data,
+            });
+        }
+
+        let (should_skip, _) = should_skip_export(&log, false);
+        assert!(
+            !should_skip,
+            "Expected to keep flight with significant gyro activity"
+        );
+    }
+
+    #[test]
+    fn test_calculate_range_empty() {
+        assert_eq!(calculate_range(&[]), 0.0);
+    }
+
+    #[test]
+    fn test_calculate_range_single_element() {
+        assert_eq!(calculate_range(&[5.0]), 0.0);
+    }
+
+    #[test]
+    fn test_calculate_range_identical_values() {
+        assert_eq!(calculate_range(&[3.0, 3.0, 3.0]), 0.0);
+    }
+
+    #[test]
+    fn test_calculate_range_normal() {
+        assert_eq!(calculate_range(&[-10.0, 0.0, 10.0]), 20.0);
     }
 
     #[test]
-    fn test_calculate_variance_single_value() {
-        let values = vec![5.0];
-        let variance = calculate_variance(&values);
-        assert_eq!(variance, 0.0);
+    fn test_calculate_range_negative_values() {
+        assert_eq!(calculate_range(&[-100.0, -50.0, -25.0]), 75.0);
     }
 
     #[test]
-    fn test_calculate_variance_empty() {
-        let values: Vec<f64> = vec![];
-        let variance = calculate_variance(&values);
-        assert_eq!(variance, 0.0);
+    fn test_calculate_range_with_nan() {
+        let result = calculate_range(&[1.0, f64::NAN, 3.0]);
+        assert!(result.is_nan(), "Expected NaN propagation with NaN input");
     }
 }

src/lib.rs

@@ -67,7 +67,8 @@
 //! ## Filtering Functions
 //! - [`should_skip_export`] - Determine if log should be skipped based on heuristics
 //! - [`has_minimal_gyro_activity`] - Detect ground tests vs actual flights
-//! - [`calculate_variance`] - Statistical helper for gyro analysis
+//! - [`calculate_range`] - Calculate gyro axis range (max - min) for scale-independent analysis
+//! - [`calculate_variance`] - DEPRECATED: Statistical helper (no longer used; kept for backward compatibility)
 //!
 //! ## Conversion Utilities
 //! - [`convert_amperage_to_amps`] - Convert raw amperage to amps

src/main.rs

@@ -314,7 +314,14 @@ fn build_command() -> Command {
         .arg(
             Arg::new("force-export")
                 .long("force-export")
-                .help("Force export of all logs, including short flights (bypasses smart filtering: <5s skip, 5-15s needs >1500fps, >15s keep)")
+                .help("Force export of all logs, bypassing smart filtering")
+                .long_help(
+                    "Force export of all logs, bypassing smart filtering.\n\n\
+                    Normal filtering behavior:\n\
+                      - Logs <5s: Always skipped\n\
+                      - Logs 5-15s: Kept if data density >1500fps\n\
+                      - Logs >15s or without duration: Checked for gyro activity (ground test detection)"
+                )
                 .action(clap::ArgAction::SetTrue),
         )
 }