用户行为日志处理策略：从文件系统到专业数据平台的演进

[26830431.7966868][4][0.013590574264526367][30398][api][1374829886.320353][init] 
  GET /foo
  {"controller"=>"foo", "action"=>"index"}

[26830431.7966868][666][2.1876697540283203][30398][api][1374829888.4944339][request_end] 
  200 OK

[request_id][user_id][time_from_request_started][process_id][app][timestamp][tagline]
  payload

# 假设您已配置好Mixpanel或Keen.io的SDK客户端
# 例如，使用Mixpanel的Ruby SDK
require 'mixpanel-ruby'

# 初始化Mixpanel客户端（通常在应用启动时完成）
# mixpanel = Mixpanel::Tracker.new("YOUR_MIXPANEL_PROJECT_TOKEN")

class ApplicationController
  def index
    request_id = generate_request_id # 假设生成一个唯一的请求ID
    user_id = current_user.id # 假设获取当前用户ID

    # 在请求开始时发送一个事件
    mixpanel.track(
      user_id,
      "Request Started",
      {
        "request_id" => request_id,
        "path" => request.path,
        "method" => request.method,
        "timestamp" => Time.now.to_f
      }
    )

    # ... 应用程序的核心逻辑 ...

    # 在请求结束时发送另一个事件
    mixpanel.track(
      user_id,
      "Request Ended",
      {
        "request_id" => request_id,
        "status_code" => response.status,
        "duration_ms" => (Time.now.to_f - start_time) * 1000 # 假设start_time已记录
      }
    )
  end

  # 其他业务逻辑...
  def purchase_item(item_id, quantity)
    user_id = current_user.id
    mixpanel.track(
      user_id,
      "Item Purchased",
      {
        "item_id" => item_id,
        "quantity" => quantity,
        "price" => get_item_price(item_id),
        "timestamp" => Time.now.to_f
      }
    )
    # ...
  end
end

#!/bin/bash

LOG_FILE="access.log"

# 定义一个函数来处理每个日志块
process_log_block() {
  local block="$1"
  # 提取第一行中的 request_id (假设是第一个方括号中的内容)
  request_id=$(echo "$block" | head -n 1 | grep -oP '^\[\K[^\]]+(?=\])' | head -n 1)
  # 提取 payload (第二行及以后)
  payload=$(echo "$block" | tail -n +2 | sed 's/^[[:space:]]*//') # 移除前导空格

  if [ -n "$request_id" ]; then
    echo "Request ID: $request_id"
    echo "Payload:"
    echo "$payload"
    echo "---"
  fi
}

# 使用awk按空行分隔日志块，并逐块处理
awk '
  BEGIN { RS = "" ; FS = "
" } # 设置记录分隔符为空行，字段分隔符为换行符
  {
    # 打印整个日志块，然后传递给bash函数处理
    print $0 | "bash -c '''process_log_block "$0"''' bash"
  }
' "$LOG_FILE"

awk -F'[][]' '
  # 检查当前行是否是日志头行（以方括号开头）
  /^\[[0-9.]+\]/ {
    # 根据用户定义的模式 [request_id][user_id]...
    # 假设 request_id 是第一个方括号内的内容
    current_request_id = $2; # awk -F'[][]' 会将方括号之间的内容作为字段

    # 读取下一行作为 payload
    getline;
    current_payload = $0;

    # 移除 payload 的前导空格
    gsub(/^[[:space:]]*/, "", current_payload);

    print "Request ID: " current_request_id;
    print "Payload: " current_payload;
    print "---";
  }
' access.log