macOS Unified Log Collection

1
2
3
4
5
6

# Enable and start the macOS System Log plugin
sudo launchctl enable system/com.logflux.agent.oslog
sudo launchctl kickstart system/com.logflux.agent.oslog

# Check status
sudo launchctl print system/com.logflux.agent.oslog

1

sudo nano /usr/local/etc/logflux-agent/plugins/oslog.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# macOS System Log Plugin Configuration
name: oslog
version: 1.0.0
source: oslog-plugin

# Agent connection
agent:
  socket_path: /var/run/logflux-agent/agent.sock

# Log filtering
filters:
  # Log levels: debug, info, default, error, fault
  levels:
    - "error"
    - "fault"
    - "default"
  
  # Time window
  since: "30 minutes ago"
  
  # Process filtering
  processes:
    include: []
    exclude:
      - "mdworker"      # Spotlight indexing
      - "mds"           # Metadata server
      - "com.apple.TimeSync"

# Metadata and labeling
metadata:
  labels:
    plugin: oslog
    os: macos
  
  # Include macOS metadata
  include_metadata: true

# Batching for efficiency
batch:
  enabled: true
  size: 100
  flush_interval: 5s

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

# Advanced macOS System Log Configuration
name: oslog
version: 1.0.0
source: oslog-plugin

# Enhanced agent settings
agent:
  socket_path: /var/run/logflux-agent/agent.sock
  connect_timeout: 30s
  max_retries: 5
  retry_delay: 10s

# Comprehensive filtering
filters:
  # Log level filtering
  levels:
    - "fault"         # Critical system errors
    - "error"         # Application errors
    - "default"       # Important notices
    - "info"          # Informational messages
    # - "debug"       # Uncomment for verbose debugging
  
  # Time-based filtering
  since: "1 hour ago"
  
  # Subsystem filtering (Apple's app organization)
  subsystems:
    include:
      - "com.apple.SecurityServer"
      - "com.apple.loginwindow"
      - "com.apple.Mail"
      - "com.apple.Safari"
      - "com.apple.networkd"
    exclude:
      - "com.apple.Spotlight"
      - "com.apple.TimeMachine"
  
  # Category filtering
  categories:
    include:
      - "security"
      - "network"
      - "authentication"
    exclude:
      - "spotlight"
      - "indexing"
  
  # Process filtering
  processes:
    include:
      - "Safari"
      - "Mail"
      - "loginwindow"
      - "SecurityServer"
    exclude:
      - "mdworker"
      - "mds"
      - "mds_stores"
      - "com.apple.metadata.mds"
      - "com.apple.Spotlight"
  
  # NSPredicate for complex filtering
  predicate: 'subsystem == "com.apple.SecurityServer" AND messageType >= 16'
  
  # Content filtering
  content_patterns:
    include:
      - "Authentication"
      - "Network"
      - "Security"
    exclude:
      - "routine"
      - "periodic"

# Performance optimization
performance:
  # Batch processing
  batch_size: 200
  flush_interval: 10s
  
  # Memory limits
  max_memory: "128MB"
  buffer_size: "1MB"
  queue_size: 5000
  
  # Message limits
  max_message_size: "64KB"
  min_message_size: 10

# Enhanced metadata
metadata:
  verbose: true
  labels:
    plugin: oslog
    os: macos
    environment: production
    datacenter: local
  
  # macOS-specific metadata
  include_metadata: true
  metadata_prefix: "macos_"
  
  # Custom field mapping
  field_mapping:
    subsystem: "macos_subsystem"
    category: "macos_category"
    process_name: "macos_process"
    process_id: "macos_pid"
    thread_id: "macos_tid"
    activity_id: "macos_activity"

# Resource limits
limits:
  max_goroutines: 20
  memory_limit: "256MB"
  cpu_limit: "1"
  
  # Rate limiting
  max_messages_per_second: 2000

# Health monitoring
health:
  check_interval: 60s
  max_parse_errors: 100
  alert_on_command_failure: true
  stats_collection: true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Monitor system errors and faults
logflux-agent-oslog -level error,fault

# Monitor specific subsystem
logflux-agent-oslog -subsystem com.apple.SecurityServer

# Monitor authentication events
logflux-agent-oslog -category authentication -since "1 hour ago"

# Monitor specific process
logflux-agent-oslog -process Safari -level error

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Security event monitoring
filters:
  levels:
    - "fault"
    - "error"
    - "default"
  
  subsystems:
    include:
      - "com.apple.SecurityServer"
      - "com.apple.authd"
      - "com.apple.opendirectoryd"
      - "com.apple.loginwindow"
  
  categories:
    include:
      - "security"
      - "authentication"
      - "authorization"
  
  predicate: 'category == "security" OR subsystem BEGINSWITH "com.apple.Security"'

metadata:
  labels:
    log_type: security
    monitoring: critical
    compliance: required

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# Application-focused monitoring
filters:
  processes:
    include:
      - "Safari"
      - "Mail"
      - "Calendar"
      - "Messages"
      - "Finder"
  
  levels:
    - "error"
    - "fault"
    - "default"
  
  content_patterns:
    include:
      - "crash"
      - "exception"
      - "error"
      - "failed"

metadata:
  labels:
    monitoring_type: applications
    focus: user_applications

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# Network and connectivity monitoring
filters:
  subsystems:
    include:
      - "com.apple.networkd"
      - "com.apple.WiFiAgent"
      - "com.apple.airport.wps"
      - "com.apple.networkserviceproxy"
  
  categories:
    include:
      - "network"
      - "wifi"
      - "ethernet"
  
  levels:
    - "error"
    - "fault"
    - "default"

metadata:
  labels:
    service: networking
    monitoring: connectivity

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Monitor all system logs
logflux-agent-oslog

# Filter by log level
logflux-agent-oslog -level error,fault

# Filter by subsystem
logflux-agent-oslog -subsystem com.apple.Mail

# Filter by process
logflux-agent-oslog -process Safari

# Time-based filtering
logflux-agent-oslog -since "30 minutes ago"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# Multiple filters combined
logflux-agent-oslog \
  -subsystem com.apple.SecurityServer \
  -level error,fault \
  -since "1 hour ago"

# Custom batch settings
logflux-agent-oslog \
  -batch-size 200 \
  -flush-interval 10s

# Disable batching for real-time
logflux-agent-oslog -no-batch

# Verbose output with statistics
logflux-agent-oslog -verbose

# Configuration file
logflux-agent-oslog -config /usr/local/etc/logflux-agent/plugins/oslog.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

{
  "subsystem": "com.apple.SecurityServer",
  "category": "security",
  "processName": "SecurityServer",
  "processID": 245,
  "threadID": 419,
  "timestamp": "2024-01-20 14:30:50.123456-0800",
  "messageType": "Error",
  "eventMessage": "Authentication failed for user admin",
  "activityIdentifier": 74565,
  "traceID": 429496729600
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

{
  "timestamp": "2024-01-20T22:30:50.123Z",
  "level": "error",
  "message": "Authentication failed for user admin",
  "node": "MacBook-Pro.local",
  "metadata": {
    "source_type": "plugin",
    "source_name": "oslog",
    "macos_subsystem": "com.apple.SecurityServer",
    "macos_category": "security",
    "macos_process": "SecurityServer",
    "macos_pid": 245,
    "macos_tid": 419,
    "macos_activity": 74565,
    "macos_trace_id": 429496729600,
    "plugin": "oslog",
    "os": "macos"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Subsystem filtering
predicate: 'subsystem == "com.apple.SecurityServer"'

# Level-based filtering (16=error, 17=fault)
predicate: 'messageType >= 16'

# Process filtering
predicate: 'processName == "Safari"'

# Category filtering
predicate: 'category == "security"'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Multiple conditions
predicate: 'subsystem == "com.apple.SecurityServer" AND messageType >= 16'

# Pattern matching
predicate: 'subsystem BEGINSWITH "com.apple.Security"'

# Content filtering
predicate: 'eventMessage CONTAINS "failed"'

# Time-based (with since parameter)
predicate: 'subsystem == "com.apple.networkd" AND category == "wifi"'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# High-throughput settings
filters:
  levels: ["error", "fault"]  # Limit to important events
  since: "10 minutes ago"     # Shorter time window

performance:
  batch_size: 500
  flush_interval: 30s
  max_memory: "512MB"
  queue_size: 10000

# Focus on specific subsystems
filters:
  subsystems:
    include:
      - "com.apple.SecurityServer"
      - "com.apple.loginwindow"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Resource-constrained settings
performance:
  batch_size: 50
  flush_interval: 60s
  max_memory: "64MB"
  queue_size: 1000
  max_message_size: "16KB"

filters:
  levels: ["fault", "error"]  # Only critical events
  since: "5 minutes ago"      # Very recent events only

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# Low-latency settings
performance:
  batch_size: 10
  flush_interval: 1s
  
batch:
  enabled: false  # Disable batching for immediate delivery

filters:
  levels: ["fault"]  # Only most critical events

1
2
3
4
5
6
7
8

# Check system log access
log show --last 1m --info >/dev/null 2>&1 && echo "Access granted" || echo "Access denied"

# Grant full disk access if needed (System Preferences > Security & Privacy)
# Required for comprehensive system log access

# Verify log command availability
which log && log help stream

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

#!/bin/bash
# check-oslog-plugin.sh

if ! launchctl print system/com.logflux.agent.oslog | grep -q "state = running"; then
    echo "CRITICAL: LogFlux macOS System Log plugin is not running"
    exit 2
fi

# Check log command access
if ! log show --last 1m --quiet >/dev/null 2>&1; then
    echo "CRITICAL: Cannot access system logs"
    exit 2
fi

# Check recent log processing
if ! log show --predicate 'subsystem == "com.logflux.agent"' --last 10m | grep -q "oslog"; then
    echo "WARNING: No recent log processing detected"
    exit 1
fi

echo "OK: LogFlux macOS System Log plugin is healthy"
exit 0

1
2
3
4
5
6
7
8

# Monitor plugin resource usage
ps aux | grep logflux-agent-oslog

# Monitor log stream performance  
log show --style compact --last 1m | wc -l

# Check system log size
log show --info --last 1h | wc -l

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# Development environment monitoring
filters:
  processes:
    include:
      - "Xcode"
      - "xcodebuild"
      - "clang"
      - "swift"
      - "iOS Simulator"
  
  levels:
    - "error"
    - "fault"
    - "default"

metadata:
  labels:
    environment: development
    monitoring: xcode

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# macOS Server monitoring
filters:
  subsystems:
    include:
      - "com.apple.server"
      - "com.apple.serverd"
      - "com.apple.SecurityServer"
      - "com.apple.networkd"
  
  levels:
    - "fault"
    - "error"
    - "default"

metadata:
  labels:
    server_role: macos_server
    monitoring: infrastructure

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# Comprehensive security monitoring
filters:
  subsystems:
    include:
      - "com.apple.SecurityServer"
      - "com.apple.authd"
      - "com.apple.opendirectoryd"
      - "com.apple.loginwindow"
  
  categories:
    include:
      - "security"
      - "authentication"
      - "authorization"
  
  predicate: 'category == "security" OR eventMessage CONTAINS "Authentication"'

metadata:
  labels:
    log_type: security_audit
    compliance: required
    retention: long_term

1
2
3
4
5
6
7
8

# Check launchd configuration
sudo launchctl print system/com.logflux.agent.oslog

# Verify plugin binary
ls -la /usr/local/bin/logflux-agent-oslog

# Check permissions
sudo launchctl enable system/com.logflux.agent.oslog

1
2
3
4
5
6
7
8

# Test log command access
log show --last 1m --info

# Check system log permissions
log stream --level info --timeout 5s

# Verify predicate syntax
log stream --predicate 'subsystem == "com.apple.SecurityServer"' --timeout 5s

1
2
3
4
5
6
7
8

# Reduce processing load
filters:
  levels: ["error", "fault"]  # Fewer log levels
  since: "5 minutes ago"      # Shorter time window

performance:
  batch_size: 200
  flush_interval: 30s

1
2
3
4
5
6
7
8

# Limit memory usage
performance:
  max_memory: "128MB"
  max_message_size: "32KB"
  queue_size: 2000

limits:
  memory_limit: "256MB"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# Enable verbose logging
export LOGFLUX_LOG_LEVEL=debug
logflux-agent-oslog -verbose

# Test configuration
logflux-agent-oslog -config /usr/local/etc/logflux-agent/plugins/oslog.yaml -verbose

# Monitor plugin logs
log stream --predicate 'subsystem == "com.logflux.agent"' --level debug

# Test specific filters
log stream --subsystem com.apple.SecurityServer --level error --timeout 10s