diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f325bdd43..25bc8a5c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,17 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: app_test + clickhouse: + image: clickhouse/clickhouse-server:26.6.1.1193 + ports: + - 8123:8123 + options: >- + --health-cmd "clickhouse-client --query 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + CLICKHOUSE_DB: hackatime_test steps: - name: Checkout code @@ -167,6 +178,11 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + CLICKHOUSE_HOST: localhost + CLICKHOUSE_PORT: 8123 + CLICKHOUSE_TEST_DATABASE: hackatime_test + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" run: | bin/rails db:create RAILS_ENV=test bin/rails db:schema:load RAILS_ENV=test @@ -179,6 +195,11 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + CLICKHOUSE_HOST: localhost + CLICKHOUSE_PORT: 8123 + CLICKHOUSE_TEST_DATABASE: hackatime_test + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" run: | bin/rails rswag:specs:swaggerize git diff --exit-code swagger/ @@ -196,6 +217,17 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: app_test + clickhouse: + image: clickhouse/clickhouse-server:26.6.1.1193 + ports: + - 8123:8123 + options: >- + --health-cmd "clickhouse-client --query 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + CLICKHOUSE_DB: hackatime_test steps: - name: Checkout code @@ -236,6 +268,11 @@ jobs: PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + CLICKHOUSE_HOST: localhost + CLICKHOUSE_PORT: 8123 + CLICKHOUSE_TEST_DATABASE: hackatime_test + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" CHROME_BIN: ${{ steps.setup-chrome.outputs.chrome-path }} CHROMEDRIVER_BIN: ${{ steps.setup-chrome.outputs.chromedriver-path }} run: | diff --git a/Gemfile b/Gemfile index c93b95afa..4f5a6d2c5 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,8 @@ gem "propshaft" gem "sqlite3", ">= 2.1" # Use PostgreSQL as the database for Wakatime gem "pg" +# ClickHouse ActiveRecord adapter - heartbeat reads + writes run on ClickHouse +gem "clickhouse-activerecord" # Use the Puma web server [https://github.com/puma/puma] gem "puma", ">= 5.0" # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails] diff --git a/Gemfile.lock b/Gemfile.lock index 28fbcfb4d..1f3e74595 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -131,6 +131,9 @@ GEM childprocess (5.1.0) logger (~> 1.5) chunky_png (1.4.0) + clickhouse-activerecord (1.6.7) + activerecord (>= 7.1, < 9.0) + bundler (>= 1.13.4) cloudflare-rails (7.0.0) actionpack (>= 7.2.0, < 8.2.0) activesupport (>= 7.2.0, < 8.2.0) @@ -594,6 +597,7 @@ DEPENDENCIES brakeman bullet capybara + clickhouse-activerecord cloudflare-rails countries debug diff --git a/app/controllers/admin/account_merger_controller.rb b/app/controllers/admin/account_merger_controller.rb index 7825e46f6..35cdaf06e 100644 --- a/app/controllers/admin/account_merger_controller.rb +++ b/app/controllers/admin/account_merger_controller.rb @@ -60,10 +60,9 @@ def format_user(user) def perform_merge(older_user, newer_user) results = [] - ActiveRecord::Base.transaction do - # 1. Move heartbeats from newer to older - results << "#{Heartbeat.where(user_id: newer_user.id).update_all(user_id: older_user.id)} heartbeats moved" + moved_heartbeats = Clickhouse::Heartbeat.where(user_id: newer_user.id).count + ActiveRecord::Base.transaction do # 2. Transfer API keys from newer to older results << "#{transfer_api_keys(older_user:, newer_user:)} API keys transferred" @@ -107,6 +106,9 @@ def perform_merge(older_user, newer_user) results << "user ##{newer_user.id} deleted" end + Clickhouse::HeartbeatWriter.merge_user_heartbeats!(older_user_id: older_user.id, newer_user_id: newer_user.id) + results << "#{moved_heartbeats} heartbeats moved" + results.join(", ") end diff --git a/app/controllers/api/admin/v1/admin_controller.rb b/app/controllers/api/admin/v1/admin_controller.rb index 08449b82e..b8264226c 100644 --- a/app/controllers/api/admin/v1/admin_controller.rb +++ b/app/controllers/api/admin/v1/admin_controller.rb @@ -38,76 +38,83 @@ def visualization_quantized quantized_query = <<-SQL WITH base_heartbeats AS ( SELECT - "time", + time, lineno, cursorpos, - date_trunc('day', to_timestamp("time")) as day_start - FROM heartbeats + toStartOfDay(toDateTime(time)) as day_start + FROM heartbeats FINAL WHERE user_id = ? AND deleted_at IS NULL - AND "time" >= ? AND "time" <= ? + AND time >= ? AND time <= ? LIMIT 1000000 ), daily_stats AS ( SELECT *, - GREATEST(1, MAX(lineno) OVER (PARTITION BY day_start)) as max_lineno, - GREATEST(1, MAX(cursorpos) OVER (PARTITION BY day_start)) as max_cursorpos + greatest(1, ifNull(MAX(lineno) OVER (PARTITION BY day_start), 1)) as max_lineno, + greatest(1, ifNull(MAX(cursorpos) OVER (PARTITION BY day_start), 1)) as max_cursorpos FROM base_heartbeats ), quantized_heartbeats AS ( SELECT *, - ROUND(2 + (("time" - extract(epoch from day_start)) / 86400) * (396)) as qx, - ROUND(2 + (1 - CAST(lineno AS decimal) / max_lineno) * (96)) as qy_lineno, - ROUND(2 + (1 - CAST(cursorpos AS decimal) / max_cursorpos) * (96)) as qy_cursorpos + round(2 + ((time - toUnixTimestamp(day_start)) / 86400) * (396)) as qx, + round(2 + (1 - toFloat64(lineno) / max_lineno) * (96)) as qy_lineno, + round(2 + (1 - toFloat64(cursorpos) / max_cursorpos) * (96)) as qy_cursorpos FROM daily_stats ) - SELECT "time", lineno, cursorpos - FROM ( - SELECT DISTINCT ON (day_start, qx, qy_lineno) "time", lineno, cursorpos - FROM quantized_heartbeats - WHERE lineno IS NOT NULL - ORDER BY day_start, qx, qy_lineno, "time" ASC - ) AS lineno_pixels - UNION - SELECT "time", lineno, cursorpos - FROM ( - SELECT DISTINCT ON (day_start, qx, qy_cursorpos) "time", lineno, cursorpos - FROM quantized_heartbeats - WHERE cursorpos IS NOT NULL - ORDER BY day_start, qx, qy_cursorpos, "time" ASC - ) AS cursorpos_pixels - UNION - SELECT "time", lineno, cursorpos - FROM ( - SELECT DISTINCT ON (day_start, qx) "time", lineno, cursorpos - FROM quantized_heartbeats - WHERE lineno IS NULL AND cursorpos IS NULL - ORDER BY day_start, qx, "time" ASC - ) AS null_pixels - ORDER BY "time" ASC + SELECT time, lineno, cursorpos FROM ( + SELECT time, lineno, cursorpos + FROM ( + SELECT time, lineno, cursorpos + FROM quantized_heartbeats + WHERE lineno IS NOT NULL + ORDER BY day_start, qx, qy_lineno, time ASC + LIMIT 1 BY day_start, qx, qy_lineno + ) AS lineno_pixels + UNION DISTINCT + SELECT time, lineno, cursorpos + FROM ( + SELECT time, lineno, cursorpos + FROM quantized_heartbeats + WHERE cursorpos IS NOT NULL + ORDER BY day_start, qx, qy_cursorpos, time ASC + LIMIT 1 BY day_start, qx, qy_cursorpos + ) AS cursorpos_pixels + UNION DISTINCT + SELECT time, lineno, cursorpos + FROM ( + SELECT time, lineno, cursorpos + FROM quantized_heartbeats + WHERE lineno IS NULL AND cursorpos IS NULL + ORDER BY day_start, qx, time ASC + LIMIT 1 BY day_start, qx + ) AS null_pixels + ) AS pixels + ORDER BY time ASC SQL daily_totals_query = <<-SQL WITH heartbeats_with_gaps AS ( SELECT - date_trunc('day', to_timestamp("time"))::date as day, - "time" - LAG("time", 1, "time") OVER (PARTITION BY date_trunc('day', to_timestamp("time")) ORDER BY "time", id) as gap - FROM heartbeats + toDate(toDateTime(time)) as day, + time - lagInFrame(time, 1, time) OVER ( + PARTITION BY toStartOfDay(toDateTime(time)) ORDER BY time, id + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) as gap + FROM heartbeats FINAL WHERE user_id = ? AND deleted_at IS NULL AND time >= ? AND time <= ? ) SELECT day, SUM(LEAST(gap, 120)) as total_seconds FROM heartbeats_with_gaps - WHERE gap IS NOT NULL GROUP BY day SQL - conn = ActiveRecord::Base.connection - quantized_result = conn.execute(ActiveRecord::Base.sanitize_sql([ quantized_query, user.id, start_epoch, end_epoch ])) - daily_totals_result = conn.execute(ActiveRecord::Base.sanitize_sql([ daily_totals_query, user.id, start_epoch, end_epoch ])) + conn = Clickhouse::Heartbeat.connection + quantized_result = conn.select_all(Clickhouse::Heartbeat.sanitize_sql([ quantized_query, user.id, start_epoch, end_epoch ])) + daily_totals_result = conn.select_all(Clickhouse::Heartbeat.sanitize_sql([ daily_totals_query, user.id, start_epoch, end_epoch ])) - daily_totals = daily_totals_result.each_with_object({}) { |row, h| h[row["day"]] = row["total_seconds"] } + daily_totals = daily_totals_result.each_with_object({}) { |row, h| h[row["day"].to_date] = row["total_seconds"].to_f.round } points_by_day = quantized_result.each_with_object({}) do |row, hash| day = Time.at(row["time"]).to_date @@ -130,8 +137,8 @@ def alt_candidates SELECT r1.user_id AS user_a_id, r2.user_id AS user_b_id, - r1.machine, - r1.ip_address, + r1.machine AS machine, + r1.ip_address AS ip_address, r1.first_seen as user_a_first_seen_on_combo, r1.last_seen as user_a_last_seen_on_combo, r2.first_seen as user_b_first_seen_on_combo, @@ -144,14 +151,13 @@ def alt_candidates ip_address, MIN(time) as first_seen, MAX(time) as last_seen - FROM heartbeats + FROM heartbeats FINAL WHERE - user_id IS NOT NULL - AND machine IS NOT NULL + machine IS NOT NULL AND ip_address IS NOT NULL AND deleted_at IS NULL AND time >= ? - GROUP BY 1, 2, 3 + GROUP BY user_id, machine, ip_address ) r1 JOIN ( @@ -161,22 +167,21 @@ def alt_candidates ip_address, MIN(time) as first_seen, MAX(time) as last_seen - FROM heartbeats + FROM heartbeats FINAL WHERE - user_id IS NOT NULL - AND machine IS NOT NULL + machine IS NOT NULL AND ip_address IS NOT NULL AND deleted_at IS NULL AND time >= ? - GROUP BY 1, 2, 3 + GROUP BY user_id, machine, ip_address ) r2 ON r1.machine = r2.machine AND r1.ip_address = r2.ip_address WHERE r1.user_id < r2.user_id LIMIT 5000 SQL - result = ActiveRecord::Base.connection.exec_query( - ActiveRecord::Base.sanitize_sql([ query, cutoff, cutoff ]) + result = Clickhouse::Heartbeat.connection.select_all( + Clickhouse::Heartbeat.sanitize_sql([ query, cutoff, cutoff ]) ) render json: { candidates: result.to_a } @@ -188,7 +193,7 @@ def active_users return render_error("invalid since parameter") if since_ts < 0 since_ts = [ since_ts, 90.days.ago.to_i ].max - render json: { user_ids: Heartbeat.where("time >= ?", since_ts).distinct.limit(50_000).pluck(:user_id) } + render json: { user_ids: Clickhouse::Heartbeat.where("time >= ?", since_ts).distinct.limit(50_000).pluck(:user_id) } end def audit_logs_counts @@ -217,7 +222,7 @@ def heartbeats_by_user_agent_segment user_id = params[:user_id].presence escaped = segment.gsub(/[\\%_]/) { |c| "\\#{c}" } - query = Heartbeat.where("user_agent ILIKE ?", "%#{escaped}%") + query = Clickhouse::Heartbeat.where("user_agent ILIKE ?", "%#{escaped}%") query = query.where(user_id: user_id) if user_id query = apply_time_range(query) or return @@ -225,7 +230,7 @@ def heartbeats_by_user_agent_segment return render json: { segment: segment, total_count: query.limit(nil).count } end - heartbeats = query.order(time: :desc).limit(limit + 1).offset(offset).to_a + heartbeats = query.order(time: :desc, id: :desc).limit(limit + 1).offset(offset).to_a has_more = heartbeats.size > limit heartbeats = heartbeats.first(limit) diff --git a/app/controllers/api/admin/v1/heartbeats_controller.rb b/app/controllers/api/admin/v1/heartbeats_controller.rb index 3cc0e9c21..b51a04df8 100644 --- a/app/controllers/api/admin/v1/heartbeats_controller.rb +++ b/app/controllers/api/admin/v1/heartbeats_controller.rb @@ -23,7 +23,7 @@ def ip_machine_pairs FROM ( SELECT user_id, machine, ip_address, MIN(time) AS first_seen, MAX(time) AS last_seen - FROM heartbeats + FROM heartbeats FINAL WHERE user_id IS NOT NULL AND machine IS NOT NULL AND ip_address IS NOT NULL @@ -34,7 +34,7 @@ def ip_machine_pairs JOIN ( SELECT user_id, machine, ip_address, MIN(time) AS first_seen, MAX(time) AS last_seen - FROM heartbeats + FROM heartbeats FINAL WHERE user_id IS NOT NULL AND machine IS NOT NULL AND ip_address IS NOT NULL @@ -46,8 +46,8 @@ def ip_machine_pairs LIMIT ? SQL - result = ActiveRecord::Base.connection.exec_query( - ActiveRecord::Base.sanitize_sql([ query, cutoff, cutoff, limit ]) + result = Clickhouse::Heartbeat.connection.select_all( + Clickhouse::Heartbeat.sanitize_sql([ query, cutoff, cutoff, limit ]) ) render json: { pairs: result.to_a } @@ -62,12 +62,12 @@ def shared_machines SELECT sms.machine, sms.machine_frequency, - ARRAY_AGG(DISTINCT u.id) AS user_ids + concat('{', arrayStringConcat(arrayMap(id -> toString(id), groupUniqArray(hb.user_id)), ','), '}') AS user_ids FROM ( SELECT machine, COUNT(user_id) AS machine_frequency FROM ( SELECT DISTINCT machine, user_id - FROM heartbeats + FROM heartbeats FINAL WHERE machine IS NOT NULL AND deleted_at IS NULL AND time > ? @@ -75,15 +75,14 @@ def shared_machines GROUP BY machine HAVING COUNT(user_id) > 1 ) AS sms - JOIN heartbeats hb ON hb.machine = sms.machine AND hb.deleted_at IS NULL AND hb.time > ? - JOIN users u ON u.id = hb.user_id + JOIN heartbeats AS hb FINAL ON hb.machine = sms.machine AND hb.deleted_at IS NULL AND hb.time > ? GROUP BY sms.machine, sms.machine_frequency ORDER BY sms.machine_frequency DESC, sms.machine ASC LIMIT ? SQL - result = ActiveRecord::Base.connection.exec_query( - ActiveRecord::Base.sanitize_sql([ query, cutoff, cutoff, limit ]) + result = Clickhouse::Heartbeat.connection.select_all( + Clickhouse::Heartbeat.sanitize_sql([ query, cutoff, cutoff, limit ]) ) render json: { machines: result.to_a } diff --git a/app/controllers/api/admin/v1/trust_level_audit_logs_controller.rb b/app/controllers/api/admin/v1/trust_level_audit_logs_controller.rb index 316535c96..4d1e929ab 100644 --- a/app/controllers/api/admin/v1/trust_level_audit_logs_controller.rb +++ b/app/controllers/api/admin/v1/trust_level_audit_logs_controller.rb @@ -10,7 +10,9 @@ class TrustLevelAuditLogsController < Api::Admin::V1::ApplicationController }.freeze def index - audit_logs = TrustLevelAuditLog.includes(:user, :changed_by).recent.limit(250) + audit_logs = TrustLevelAuditLog + .includes(user: :email_addresses, changed_by: :email_addresses) + .recent.limit(250) if params[:user_id].present? user = User.find_by(id: params[:user_id]) diff --git a/app/controllers/api/hackatime/v1/hackatime_controller.rb b/app/controllers/api/hackatime/v1/hackatime_controller.rb index c954c04d9..4f26c73cf 100644 --- a/app/controllers/api/hackatime/v1/hackatime_controller.rb +++ b/app/controllers/api/hackatime/v1/hackatime_controller.rb @@ -34,7 +34,7 @@ def push_heartbeats def status_bar_today Time.use_zone(@user.timezone) do - total_seconds = @user.heartbeats.today.duration_seconds + total_seconds = Clickhouse::Heartbeat.for_user(@user).today.duration_seconds result = { data: { grand_total: { @@ -69,7 +69,7 @@ def stats_last_7_days start_time = (Time.current - 7.days).beginning_of_day end_time = Time.current.end_of_day - heartbeats = @user.heartbeats.where(time: start_time.to_i..end_time.to_i) + heartbeats = Clickhouse::Heartbeat.for_user(@user).where(time: start_time.to_i..end_time.to_i) total_seconds = heartbeats.duration_seconds.to_i days_covered = heartbeats.pluck(:time).map { |ts| Time.at(ts).in_time_zone(@user.timezone).to_date }.uniq.length daily_average = days_covered > 0 ? (total_seconds.to_f / days_covered).round(1) : 0 @@ -166,7 +166,7 @@ def handle_heartbeat(heartbeat_array) ) result.items.map do |item| - next [ item.heartbeat.attributes, 201 ] if item.status == :accepted + next [ item.heartbeat, 201 ] if item.status == :accepted error = item.error report_error( error, diff --git a/app/controllers/api/v1/authenticated/heartbeats_controller.rb b/app/controllers/api/v1/authenticated/heartbeats_controller.rb index 9be5c7ac1..cb728f149 100644 --- a/app/controllers/api/v1/authenticated/heartbeats_controller.rb +++ b/app/controllers/api/v1/authenticated/heartbeats_controller.rb @@ -3,10 +3,10 @@ module V1 module Authenticated class HeartbeatsController < ApplicationController def latest - heartbeat = current_user.heartbeats - .where.not(source_type: :test_entry) - .order(time: :desc) - .first + heartbeat = Clickhouse::Heartbeat.for_user(current_user) + .where.not(source_type: :test_entry) + .order(time: :desc, id: :desc) + .first if heartbeat render json: { diff --git a/app/controllers/api/v1/authenticated/hours_controller.rb b/app/controllers/api/v1/authenticated/hours_controller.rb index 92f8ca874..e702f56f3 100644 --- a/app/controllers/api/v1/authenticated/hours_controller.rb +++ b/app/controllers/api/v1/authenticated/hours_controller.rb @@ -6,9 +6,9 @@ def index start_date = params[:start_date]&.to_date || 7.days.ago.to_date end_date = params[:end_date]&.to_date || Date.current - total_seconds = current_user.heartbeats - .where(time: start_date.beginning_of_day.to_i..end_date.end_of_day.to_i) - .duration_seconds + total_seconds = Clickhouse::Heartbeat.for_user(current_user) + .where(time: start_date.beginning_of_day.to_i..end_date.end_of_day.to_i) + .duration_seconds render json: { start_date: start_date, diff --git a/app/controllers/api/v1/badges_controller.rb b/app/controllers/api/v1/badges_controller.rb index af4e88ff7..6a843efc0 100644 --- a/app/controllers/api/v1/badges_controller.rb +++ b/app/controllers/api/v1/badges_controller.rb @@ -16,13 +16,13 @@ def show project_name = resolve_project_name(user, params[:project]) return render_not_found_json("Project not found") unless project_name - seconds = user.heartbeats.where(project: project_name).duration_seconds + seconds = Clickhouse::Heartbeat.for_user(user).where(project: project_name).duration_seconds return head :bad_request if seconds <= 0 # Handle aliases (comma-separated project names to sum) if params[:aliases].present? alias_names = params[:aliases].split(",").map(&:strip) - [ project_name ] - seconds += user.heartbeats.where(project: alias_names).duration_seconds + seconds += Clickhouse::Heartbeat.for_user(user).where(project: alias_names).duration_seconds end label = params[:label] || "hackatime" @@ -48,7 +48,8 @@ def find_user(identifier) # Resolve owner/repo format to a project name via ProjectRepoMapping def resolve_project_name(user, project_param) return nil if project_param.blank? - return project_param if user.heartbeats.where(project: project_param).exists? + project_scope = Clickhouse::Heartbeat.for_user(user).where(project: project_param) + return project_param if Clickhouse::Heartbeat.safe_exists?(project_scope) if project_param.include?("/") owner, name = project_param.split("/", 2) diff --git a/app/controllers/api/v1/my/heartbeats_controller.rb b/app/controllers/api/v1/my/heartbeats_controller.rb index b413d1edb..3f88e6ecb 100644 --- a/app/controllers/api/v1/my/heartbeats_controller.rb +++ b/app/controllers/api/v1/my/heartbeats_controller.rb @@ -3,7 +3,7 @@ class Api::V1::My::HeartbeatsController < ApplicationController before_action :ensure_authenticated! def most_recent - scope = current_user.heartbeats.order(time: :desc) + scope = Clickhouse::Heartbeat.for_user(current_user).order(time: :desc, id: :desc) if params[:source_type].present? scope = scope.where(source_type: params[:source_type]) @@ -17,7 +17,7 @@ def most_recent render json: { has_heartbeat: heartbeat.present?, - heartbeat: heartbeat, + heartbeat: heartbeat && heartbeat_json(heartbeat), editor: heartbeat&.editor, time_ago: heartbeat.present? ? time_ago_in_words(Time.at(heartbeat.time)) + " ago" : nil } @@ -27,18 +27,22 @@ def index start_time = params[:start_time].present? ? Time.parse(params[:start_time]) : Time.current.beginning_of_day end_time = params[:end_time].present? ? Time.parse(params[:end_time]) : Time.current.end_of_day - heartbeats = current_user.heartbeats + heartbeats = Clickhouse::Heartbeat.for_user(current_user) .where("time >= ? AND time <= ?", start_time.to_f, end_time.to_f) - .order(time: :asc) + .order(time: :asc, id: :asc) render json: { start_time: start_time, end_time: end_time, - total_seconds: heartbeats.duration_seconds, heartbeats: heartbeats + total_seconds: heartbeats.duration_seconds, heartbeats: heartbeats.map { |hb| heartbeat_json(hb) } } end private + def heartbeat_json(heartbeat) + heartbeat.as_json(except: %w[version]) + end + def ensure_authenticated! api_header = request.headers["Authorization"] raw_token = api_header&.split(" ")&.last diff --git a/app/controllers/api/v1/stats_controller.rb b/app/controllers/api/v1/stats_controller.rb index 4b169f834..2eae0f701 100644 --- a/app/controllers/api/v1/stats_controller.rb +++ b/app/controllers/api/v1/stats_controller.rb @@ -12,7 +12,7 @@ def show end_date = parse_date_param(:end_date, default: Date.today.end_of_day, boundary: :end) return if performed? - query = Heartbeat.where(time: start_date..end_date) + query = Clickhouse::Heartbeat.where(time: start_date..end_date) if params[:username].present? user = User.lookup_by_identifier(params[:username]) @@ -39,7 +39,7 @@ def user_stats # /api/v1/users/current/stats?filter_by_project=harbor,high-seas filter_by_projects = params[:filter_by_project].presence&.split(",") filter_by_categories = params[:filter_by_category].presence&.split(",") - scope = filter_by_projects ? @user.heartbeats.where(project: filter_by_projects) : nil + scope = filter_by_projects ? Clickhouse::Heartbeat.for_user(@user).where(project: filter_by_projects) : nil enabled_features = params[:features]&.split(",")&.map(&:to_sym) || %i[languages] @@ -69,14 +69,14 @@ def user_stats summary = WakatimeService.new(**service_params).generate_summary else if params[:total_seconds] == "true" - query = Heartbeat.where(user_id: @user.id).where("time >= ? AND time < ?", start_date.to_f, end_date.to_f) + query = Clickhouse::Heartbeat.for_user(@user).where("time >= ? AND time < ?", start_date.to_f, end_date.to_f) query = query.where(project: filter_by_projects) if filter_by_projects query = query.where(category: filter_by_categories) if filter_by_categories total_seconds = if params[:boundary_aware] == "true" excluded = [ "browsing", "meeting", "communicating" ] excluded << "ai coding" if no_ai_coding - Heartbeat.duration_seconds_boundary_aware( + Clickhouse::Heartbeat.duration_seconds_boundary_aware( query, start_date.to_f, end_date.to_f, @@ -95,8 +95,8 @@ def user_stats end if params[:features]&.include?("projects") && params[:filter_by_project].present? - heartbeats = @user.heartbeats.coding_only.with_valid_timestamps - .where(time: start_date..end_date, project: filter_by_projects) + heartbeats = Clickhouse::Heartbeat.for_user(@user).coding_only.with_valid_timestamps + .where(time: start_date..end_date, project: filter_by_projects) summary[:unique_total_seconds] = unique_heartbeat_seconds(heartbeats) end @@ -118,7 +118,7 @@ def user_spans end_date = parse_datetime_param(:end_date, default: Date.today.end_of_day) return if performed? - heartbeats = @user.heartbeats.where(time: start_date.to_f..end_date.to_f) + heartbeats = Clickhouse::Heartbeat.for_user(@user).where(time: start_date.to_f..end_date.to_f) heartbeats = heartbeats.where(project: params[:project]) if params[:project].present? heartbeats = heartbeats.where(project: params[:filter_by_project].split(",")) if params[:project].blank? && params[:filter_by_project].present? diff --git a/app/controllers/concerns/api/admin/v1/user_utilities.rb b/app/controllers/concerns/api/admin/v1/user_utilities.rb index a8cb379fa..24a5241d5 100644 --- a/app/controllers/concerns/api/admin/v1/user_utilities.rb +++ b/app/controllers/concerns/api/admin/v1/user_utilities.rb @@ -59,7 +59,7 @@ def search_users_fuzzy def get_users_by_ip return render_error("bro dont got the ip") if params[:ip].blank? - result = Heartbeat.where(ip_address: params[:ip]).select(:ip_address, :user_id, :machine, :user_agent).distinct + result = Clickhouse::Heartbeat.where(ip_address: params[:ip]).select(:ip_address, :user_id, :machine, :user_agent).distinct render json: { users: result.map { |u| { @@ -75,7 +75,7 @@ def get_users_by_ip def get_users_by_machine return render_error("bro dont got the machine") if params[:machine].blank? - result = Heartbeat.where(machine: params[:machine]).select(:user_id, :machine).distinct + result = Clickhouse::Heartbeat.where(machine: params[:machine]).select(:user_id, :machine).distinct render json: { users: result.map { |u| { user_id: u.user_id, machine: u.machine } } } end @@ -83,7 +83,7 @@ def user_info user = find_user_by_id return unless user - valid = user.heartbeats.where("CASE WHEN time > 1000000000000 THEN time / 1000 ELSE time END BETWEEN ? AND ?", Time.utc(2000, 1, 1).to_i, Time.utc(2100, 1, 1).to_i) + valid = Clickhouse::Heartbeat.for_user(user).where("CASE WHEN time > 1000000000000 THEN time / 1000 ELSE time END BETWEEN ? AND ?", Time.utc(2000, 1, 1).to_i, Time.utc(2100, 1, 1).to_i) lht = valid.maximum(:time) lht /= 1000 if lht && lht > 1000000000000 @@ -112,7 +112,7 @@ def user_info total_coding_time: valid.duration_seconds || 0, languages_used: valid.distinct.pluck(:language).compact.count, projects_worked_on: valid.distinct.pluck(:project).compact.count, - days_active: valid.distinct.count("DATE(to_timestamp(CASE WHEN time > 1000000000000 THEN time / 1000 ELSE time END))") + days_active: valid.distinct.count("toDate(toDateTime(CASE WHEN time > 1000000000000 THEN time / 1000 ELSE time END))") } } } @@ -133,8 +133,7 @@ def user_stats end_time = date.end_of_day.utc end - heartbeats = user.heartbeats.where(time: start_time.to_i..end_time.to_i).order(:time) - + heartbeats = Clickhouse::Heartbeat.for_user(user).where(time: start_time.to_i..end_time.to_i).order(:time, :id) render json: { user_id: user.id, username: user.display_name, @@ -177,7 +176,7 @@ def user_projects user = find_user_by_id return unless user - base_heartbeats = user.heartbeats.where.not(project: nil) + base_heartbeats = Clickhouse::Heartbeat.for_user(user).where.not(project: nil) if params[:start_date].present? || params[:end_date].present? range = parse_default_time_range or return @@ -187,7 +186,7 @@ def user_projects project_stats = base_heartbeats .select(:project, "COUNT(*) as heartbeat_count", "MIN(time) as first_heartbeat", "MAX(time) as last_heartbeat", - "ARRAY_AGG(DISTINCT language) FILTER (WHERE language IS NOT NULL) as languages") + "groupUniqArrayIf(assumeNotNull(language), language IS NOT NULL) as languages") .group(:project).order(Arel.sql("COUNT(*) DESC")) durations = base_heartbeats.group(:project).duration_seconds @@ -298,15 +297,14 @@ def user_heartbeats limit = (params[:limit] || 1000).to_i.clamp(1, 5_000) offset = (params[:offset] || 0).to_i.clamp(0, Float::INFINITY) - query = user.heartbeats + query = Clickhouse::Heartbeat.for_user(user) query = apply_time_range(query) or return %i[project language entity editor machine].each do |f| query = query.where(f => params[f]) if params[f].present? end total_count = query.count - source_types = Heartbeat.source_types.invert - rows = query.order(time: :asc).limit(limit).offset(offset).pluck(*HEARTBEAT_RESPONSE_COLUMNS) + rows = query.order(time: :asc, id: :asc).limit(limit).offset(offset).pluck(*HEARTBEAT_RESPONSE_COLUMNS) ja4s_by_id = Ja4.where(id: rows.filter_map(&:last).uniq).index_by(&:id) heartbeats = rows.map do |id, time, lineno, cursorpos, is_write, project, language, entity, branch, category, editor, machine, user_agent, ip_address, lines, source_type, ja4_id| { @@ -326,7 +324,7 @@ def user_heartbeats ip_address: ip_address, ja4: ja4s_by_id[ja4_id]&.then { |ja4| { fingerprint: ja4.fingerprint, name: ja4.name } }, lines: lines, - source_type: source_types[source_type] || source_type + source_type: source_type } end @@ -348,10 +346,10 @@ def user_heartbeat_values limit = (params[:limit] || 5000).to_i.clamp(1, 5000) - query = user.heartbeats + query = Clickhouse::Heartbeat.for_user(user) query = apply_time_range(query) or return - quoted_column = Heartbeat.connection.quote_column_name(column_name) + quoted_column = Clickhouse::Heartbeat.connection.quote_column_name(column_name) values = query.where.not(column_name => nil).distinct .order(Arel.sql("#{quoted_column} ASC")) .limit(limit).pluck(column_name).reject(&:empty?) diff --git a/app/controllers/inertia_controller.rb b/app/controllers/inertia_controller.rb index 56d602f6b..ad55cd541 100644 --- a/app/controllers/inertia_controller.rb +++ b/app/controllers/inertia_controller.rb @@ -155,8 +155,8 @@ def inertia_footer_props git_version: Rails.application.config.git_version, commit_link: Rails.application.config.commit_link, server_start_time_ago: h.time_ago_in_words(Rails.application.config.server_start_time), - heartbeat_recent_count: Heartbeat.recent_count, - heartbeat_recent_imported_count: Heartbeat.recent_imported_count, + heartbeat_recent_count: Clickhouse::Heartbeat.recent_count, + heartbeat_recent_imported_count: Clickhouse::Heartbeat.recent_imported_count, active_users_graph: hours } diff --git a/app/controllers/my/project_repo_mappings_controller.rb b/app/controllers/my/project_repo_mappings_controller.rb index 30b6908a8..6e45a33a8 100644 --- a/app/controllers/my/project_repo_mappings_controller.rb +++ b/app/controllers/my/project_repo_mappings_controller.rb @@ -45,7 +45,7 @@ def archive def show project_name = CGI.unescape(params[:project_name]) mapping = current_user.project_repo_mappings.find_by(project_name: project_name) - first_heartbeat = current_user.heartbeats.where(project: project_name).minimum(:time) + first_heartbeat = heartbeats_scope.where(project: project_name).minimum(:time) since_date = first_heartbeat ? Time.at(first_heartbeat).to_date.strftime("%-m/%-d/%Y") : nil share_url = if mapping&.public_shared_at.present? && current_user.username.present? @@ -100,6 +100,7 @@ def set_project_repo_mapping @project_repo_mapping = current_user.project_repo_mappings.find_or_create_by!(project_name: params[:project_name]) end + def heartbeats_scope = Clickhouse::Heartbeat.for_user(current_user) def project_repo_mapping_params = params.require(:project_repo_mapping).permit(:repo_url) def show_archived? = params[:show_archived] == "true" def selected_interval = params[:interval] @@ -116,7 +117,7 @@ def project_durations_cache_key def sanitized_cache_date(value) = value.to_s.gsub(/[^0-9-]/, "")[0, 10].presence - # Builds the data needed for either projects_payload or rollup_projects_payload: + # Builds the data needed for projects_payload: # scoped mappings, archived name set, and latest commit timestamps by repo id. def projects_context(archived:) mappings = current_user.project_repo_mappings.includes(:repository) @@ -134,7 +135,7 @@ def projects_payload(archived:) mappings_by_name, archived_names, latest_user_commit_at_by_repo_id = projects_context(archived: archived) cached = Rails.cache.fetch(project_durations_cache_key, expires_in: 1.minute) do - hb = current_user.heartbeats.filter_by_time_range(selected_interval, params[:from], params[:to]) + hb = heartbeats_scope.filter_by_time_range(selected_interval, params[:from], params[:to]) { durations: hb.group(:project).duration_seconds, total_time: hb.duration_seconds } end @@ -149,9 +150,6 @@ def projects_payload(archived:) end def projects_data_for_index(archived:) - return empty_projects_payload unless current_user.heartbeats.exists? - return rollup_projects_payload(archived: archived) if rollup_projects_path? - InertiaRails.defer { projects_payload(archived: archived) } end @@ -159,32 +157,6 @@ def empty_projects_payload { total_time_seconds: 0, total_time_label: format_duration(0), has_activity: false, projects: [] } end - def rollup_projects_path? = selected_interval.blank? && params[:from].blank? && params[:to].blank? - - def rollup_projects_payload(archived:) - rollups = DashboardRollup - .where(user_id: current_user.id, dimension: DashboardRollup::PROJECT_DETAILS_DIMENSION, bucket_value_present: true) - .to_a - - DashboardRollupRefreshJob.schedule_for(current_user.id, wait: 0.seconds) if DashboardRollup.dirty?(current_user.id) || rollups.empty? - return InertiaRails.defer { projects_payload(archived: archived) } if rollups.empty? - - mappings_by_name, archived_names, latest_user_commit_at_by_repo_id = projects_context(archived: archived) - - projects = rollups.filter_map do |rollup| - project_key = rollup.bucket - next if project_key.blank? - next if archived_names.key?(project_key) != archived - - duration = rollup.total_seconds.to_i - next if duration <= 0 - - project_summary_payload(project_key, duration, mappings_by_name[project_key], latest_user_commit_at_by_repo_id) - end.sort_by { |project| -project[:duration_seconds] } - - build_projects_payload(projects) - end - def project_summary_payload(project_key, duration, mapping, latest_user_commit_at_by_repo_id) display_name = project_key.presence || "Unknown" broken = broken_project_name?(project_key, display_name) @@ -241,13 +213,17 @@ def repository_payload(repository, latest_user_commit_at_by_repo_id = {}) def project_count(archived) archived_names = current_user.project_repo_mappings.archived.pluck(:project_name) - hb = current_user.heartbeats.filter_by_time_range(selected_interval, params[:from], params[:to]) + hb = heartbeats_scope.filter_by_time_range(selected_interval, params[:from], params[:to]) projects = hb.select(:project).distinct.pluck(:project) projects.count { |proj| archived_names.include?(proj) == archived } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + archived ? current_user.project_repo_mappings.archived.count : current_user.project_repo_mappings.active.count end def project_detail_payload(project_name) - hb = current_user.heartbeats.where(project: project_name) + hb = heartbeats_scope.where(project: project_name) .filter_by_time_range(selected_interval, params[:from], params[:to]) stats = ProjectStatsService.new(hb).call @@ -259,7 +235,7 @@ def project_detail_payload(project_name) def project_activity_graph(project_name) snapshot = DashboardData::Snapshots.activity_graph_snapshot( - user: current_user, scope: current_user.heartbeats.where(project: project_name) + user: current_user, scope: heartbeats_scope.where(project: project_name) ) DashboardData::Snapshots.activity_graph_result( start_date: snapshot[:start_date], diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb index 336483656..1f02c1af3 100644 --- a/app/controllers/profiles_controller.rb +++ b/app/controllers/profiles_controller.rb @@ -45,7 +45,7 @@ def project return head :not_found unless mapping&.public_shared_at.present? h = ApplicationController.helpers - hb = @user.heartbeats.where(project: project_name) + hb = Clickhouse::Heartbeat.for_user(@user).where(project: project_name) stats = ProjectStatsService.new(hb).call( only: %i[total_time file_count language_stats language_colors file_stats branch_stats] ) @@ -135,14 +135,8 @@ def public_profile_og_stats def public_profile_og_heatmap return nil unless @user.allow_public_stats_lookup - rollup = DashboardRollup.find_by(user_id: @user.id, dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - duration_by_date = rollup&.payload&.fetch("duration_by_date", nil) - return nil if duration_by_date.blank? - - duration_by_date.each_with_object({}) do |(date, seconds), out| - key = (date.is_a?(Date) ? date.iso8601 : date.to_date.iso8601 rescue date.to_s) - out[key] = seconds.to_i - end + duration_by_date = DashboardStats.new(user: @user).activity_graph_data[:duration_by_date] + duration_by_date.presence end def profile_summary_payload diff --git a/app/controllers/settings/base_controller.rb b/app/controllers/settings/base_controller.rb index 474fc40b6..af4f9cda8 100644 --- a/app/controllers/settings/base_controller.rb +++ b/app/controllers/settings/base_controller.rb @@ -112,7 +112,7 @@ def goal_options goal_languages = [] goal_projects = project_list.map { |p| p[:display_name] } - @user.heartbeats.distinct.pluck(:language, :project).each do |language, project| + Clickhouse::Heartbeat.for_user(@user).distinct.pluck(:language, :project).each do |language, project| categorized = language&.categorize_language goal_languages << categorized if categorized.present? goal_projects << project if project.present? diff --git a/app/controllers/settings/imports_exports_controller.rb b/app/controllers/settings/imports_exports_controller.rb index 9c775bd21..1386958ee 100644 --- a/app/controllers/settings/imports_exports_controller.rb +++ b/app/controllers/settings/imports_exports_controller.rb @@ -21,10 +21,12 @@ def section_props { data_export: InertiaRails.defer { + heartbeats = Clickhouse::Heartbeat.for_user(@user) + { - total_heartbeats: number_with_delimiter(@user.heartbeats.count), - total_coding_time: @user.heartbeats.duration_simple, - heartbeats_last_7_days: number_with_delimiter(@user.heartbeats.where("time >= ?", 7.days.ago.to_f).count), + total_heartbeats: number_with_delimiter(safe_heartbeat_count(heartbeats)), + total_coding_time: Clickhouse::Heartbeat.duration_simple(heartbeats), + heartbeats_last_7_days: number_with_delimiter(safe_heartbeat_count(heartbeats.where("time >= ?", 7.days.ago.to_f))), is_restricted: (@user.trust_level == "red") } }, @@ -40,4 +42,12 @@ def section_props end def export_cooldown_minutes = My::HeartbeatsController::EXPORT_COOLDOWN.in_minutes.to_i + + def safe_heartbeat_count(scope) + scope.count + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + 0 + end end diff --git a/app/controllers/static_pages_controller.rb b/app/controllers/static_pages_controller.rb index 669c7270a..9e4f626bd 100644 --- a/app/controllers/static_pages_controller.rb +++ b/app/controllers/static_pages_controller.rb @@ -14,7 +14,7 @@ def index return redirect_to "/my/projects?interval=custom&from=#{d}&to=#{d}" if d end - @show_wakatime_setup_notice = true if !current_user.heartbeats.exists? || params[:show_wakatime_setup_notice] + @show_wakatime_setup_notice = true if !Clickhouse::Heartbeat.safe_exists?(Clickhouse::Heartbeat.for_user(current_user)) || params[:show_wakatime_setup_notice] render inertia: "Home/SignedIn", props: signed_in_props else @@ -74,13 +74,12 @@ def set_homepage_seo_content end def signed_in_props - dashboard_stats = initial_dashboard_stats_prop { flavor_text: @flavor_text.to_s, trust_level_red: current_user&.trust_level == "red", show_wakatime_setup_notice: !!@show_wakatime_setup_notice, github_uid_blank: current_user&.github_uid.blank?, - dashboard_stats: dashboard_stats || InertiaRails.defer { dashboard_stats_payload } + dashboard_stats: InertiaRails.defer { dashboard_stats_payload } } end @@ -114,8 +113,4 @@ def programming_goals_progress_data ProgrammingGoalsProgressService.new(user: current_user, goals: goals).call end end - - def initial_dashboard_stats_prop - dashboard_stats_payload if dashboard_stats.rollup_eligible? && dashboard_stats.rollups_available? && dashboard_stats.rollup_total_row - end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 4aa5e5681..2fd12b9d0 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -60,8 +60,8 @@ def short_time_detailed(time) def time_in_emoji(duration) = CLOCK_EMOJIS[(duration.to_i / 1800) % CLOCK_EMOJIS.length] def human_interval_name(key, from: nil, to: nil) - if key.present? && Heartbeat.respond_to?(:humanize_range) && Heartbeat::RANGES.key?(key.to_sym) - Heartbeat.humanize_range(Heartbeat::RANGES[key.to_sym][:calculate].call) + if key.present? && Clickhouse::Heartbeat.respond_to?(:humanize_range) && Clickhouse::Heartbeat::RANGES.key?(key.to_sym) + Clickhouse::Heartbeat.humanize_range(Clickhouse::Heartbeat::RANGES[key.to_sym][:calculate].call) elsif from.present? && to.present? "#{from} to #{to}" else diff --git a/app/jobs/cache/active_projects_job.rb b/app/jobs/cache/active_projects_job.rb index 863384c01..0c8cc45ad 100644 --- a/app/jobs/cache/active_projects_job.rb +++ b/app/jobs/cache/active_projects_job.rb @@ -6,24 +6,27 @@ class Cache::ActiveProjectsJob < Cache::ActivityJob def cache_expiration = 15.minutes def calculate - sql = ProjectRepoMapping.sanitize_sql_array([ <<~SQL, Heartbeat.source_types[:direct_entry], 5.minutes.ago.to_f ]) - WITH recent AS MATERIALIZED ( - SELECT user_id, project, time - FROM heartbeats - WHERE source_type = ? - AND deleted_at IS NULL - AND time > ? - ) - SELECT DISTINCT ON (recent.user_id) project_repo_mappings.*, recent.user_id - FROM project_repo_mappings - INNER JOIN recent - ON recent.project = project_repo_mappings.project_name - AND recent.user_id = project_repo_mappings.user_id - INNER JOIN users ON users.id = recent.user_id - WHERE project_repo_mappings.archived_at IS NULL - ORDER BY recent.user_id, recent.time DESC - SQL + recent_projects = Clickhouse::Heartbeat + .where(source_type: :direct_entry) + .where("time > ?", 5.minutes.ago.to_f) + .group(:user_id, :project) + .pluck(:user_id, :project, Arel.sql("max(time) AS latest_time")) + return {} if recent_projects.empty? - ProjectRepoMapping.find_by_sql(sql).index_by(&:user_id) + user_ids = recent_projects.map(&:first).uniq + project_names = recent_projects.map { |_, project, _| project }.compact.uniq + + mappings = ProjectRepoMapping + .where(user_id: user_ids, project_name: project_names) + .where(archived_at: nil) + .index_by { |mapping| [ mapping.user_id, mapping.project_name ] } + + # Latest recent heartbeat per user that has an active repo mapping. + recent_projects.sort_by { |_, _, time| -time.to_f }.each_with_object({}) do |(user_id, project, _), result| + next if result.key?(user_id) + + mapping = mappings[[ user_id, project ]] + result[user_id] = mapping if mapping + end end end diff --git a/app/jobs/cache/active_users_graph_data_job.rb b/app/jobs/cache/active_users_graph_data_job.rb index 0dd99b774..e357f90a1 100644 --- a/app/jobs/cache/active_users_graph_data_job.rb +++ b/app/jobs/cache/active_users_graph_data_job.rb @@ -4,12 +4,21 @@ class Cache::ActiveUsersGraphDataJob < Cache::ActivityJob private def calculate - hours = Heartbeat.coding_only.with_valid_timestamps + inner = Clickhouse::Heartbeat.coding_only.with_valid_timestamps .where("time > ?", 24.hours.ago.to_f).where("time < ?", Time.current.to_f) - .select("(EXTRACT(EPOCH FROM to_timestamp(time))::bigint / 3600 * 3600) as hour, COUNT(DISTINCT user_id) as count") - .group("hour").order("hour DESC") + .select(Arel.sql("intDiv(toInt64(round(time)), 3600) * 3600 AS hour, COUNT(DISTINCT user_id) AS count")) + .group(Arel.sql("hour")).order(Arel.sql("hour DESC")) + .to_sql - top = hours.max_by(&:count)&.count || 1 - hours.map { |h| { hour: Time.at(h.hour), users: h.count, height: (h.count.to_f / top * 100).round } } + hours = begin + Clickhouse::Heartbeat.connection.select_all(inner).map { |row| { hour: row["hour"].to_i, count: row["count"].to_i } } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + [] + end + + top = hours.max_by { |h| h[:count] }&.fetch(:count) || 1 + hours.map { |h| { hour: Time.at(h[:hour]), users: h[:count], height: (h[:count].to_f / top * 100).round } } end end diff --git a/app/jobs/cache/currently_hacking_count_job.rb b/app/jobs/cache/currently_hacking_count_job.rb index ca87a8017..99608279a 100644 --- a/app/jobs/cache/currently_hacking_count_job.rb +++ b/app/jobs/cache/currently_hacking_count_job.rb @@ -6,8 +6,9 @@ class Cache::CurrentlyHackingCountJob < Cache::ActivityJob def cache_expiration = 1.minute def calculate - count = Heartbeat.joins(:user).where(source_type: :direct_entry).coding_only - .where("time > ?", 5.minutes.ago.to_f).select("DISTINCT user_id").count + count = Clickhouse::Heartbeat + .where(source_type: :direct_entry).coding_only + .where("time > ?", 5.minutes.ago.to_f).distinct.count(:user_id) { count: count } end end diff --git a/app/jobs/cache/currently_hacking_job.rb b/app/jobs/cache/currently_hacking_job.rb index 50e846230..0aaf681f5 100644 --- a/app/jobs/cache/currently_hacking_job.rb +++ b/app/jobs/cache/currently_hacking_job.rb @@ -6,18 +6,19 @@ class Cache::CurrentlyHackingJob < Cache::ActivityJob def cache_expiration = 5.minutes def calculate - recent_heartbeats = Heartbeat.joins(:user) + latest_project_by_user = Clickhouse::Heartbeat .where(source_type: :direct_entry).coding_only .where("time > ?", 5.minutes.ago.to_f) - .select("DISTINCT ON (user_id) user_id, project, time, users.*") - .order("user_id, time DESC") - .includes(user: [ :project_repo_mappings, :email_addresses ]) - .index_by(&:user_id) + .group(:user_id) + .pluck(Arel.sql("user_id, argMax(ifNull(project, ''), tuple(time, id))")) + .to_h + + users = User.where(id: latest_project_by_user.keys) + .includes(:project_repo_mappings, :email_addresses).to_a - users = recent_heartbeats.values.map(&:user) active_projects = {} users.each do |user| - mapping = user.project_repo_mappings.find { |p| p.project_name == recent_heartbeats[user.id]&.project } + mapping = user.project_repo_mappings.find { |p| p.project_name == latest_project_by_user[user.id] } active_projects[user.id] = mapping&.archived? ? nil : mapping end diff --git a/app/jobs/cache/heartbeat_counts_job.rb b/app/jobs/cache/heartbeat_counts_job.rb index 396df3b90..a73af80ff 100644 --- a/app/jobs/cache/heartbeat_counts_job.rb +++ b/app/jobs/cache/heartbeat_counts_job.rb @@ -4,11 +4,15 @@ class Cache::HeartbeatCountsJob < Cache::ActivityJob private def calculate - direct = Heartbeat.source_types.fetch("direct_entry") - recent_count, recent_imported_count = Heartbeat.recent.pluck( + direct = Clickhouse::Heartbeat.source_types.fetch("direct_entry") + recent_count, recent_imported_count = Clickhouse::Heartbeat.recent.pluck( Arel.sql("COUNT(*)"), - Arel.sql("COUNT(*) FILTER (WHERE source_type != #{direct})") + Arel.sql("countIf(source_type != #{direct})") ).first { recent_count:, recent_imported_count: } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + { recent_count: 0, recent_imported_count: 0 } end end diff --git a/app/jobs/cache/home_stats_job.rb b/app/jobs/cache/home_stats_job.rb index 318fe3b8d..41eb13cb7 100644 --- a/app/jobs/cache/home_stats_job.rb +++ b/app/jobs/cache/home_stats_job.rb @@ -4,9 +4,26 @@ class Cache::HomeStatsJob < Cache::ActivityJob private def calculate - users_tracked, seconds_tracked = DashboardRollup - .where(dimension: DashboardRollup::TOTAL_DIMENSION).where(total_seconds: 1..) - .pick(Arel.sql("COUNT(*), COALESCE(SUM(total_seconds), 0)")) - { users_tracked: users_tracked || 0, seconds_tracked: seconds_tracked || 0 } + timeout = Clickhouse::Heartbeat.heartbeat_timeout_duration.to_i + lag_sql = "lagInFrame(time, 1, time) OVER (" \ + "PARTITION BY user_id ORDER BY time, id " \ + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + relation_sql = Clickhouse::Heartbeat.with_valid_timestamps.select(:user_id, :id, :time).to_sql + + row = Clickhouse::Heartbeat.connection.select_one(<<~SQL.squish) + SELECT countIf(duration >= 1) AS users_tracked, + sumIf(duration, duration >= 1) AS seconds_tracked + FROM ( + SELECT user_id, round(SUM(diff)) AS duration + FROM ( + SELECT user_id, + least(time - #{lag_sql}, #{timeout}) AS diff + FROM (#{relation_sql}) AS home_stats_heartbeats + ) AS diffs + GROUP BY user_id + ) AS user_durations + SQL + + { users_tracked: row["users_tracked"].to_i, seconds_tracked: row["seconds_tracked"].to_i } end end diff --git a/app/jobs/cache/minutes_logged_job.rb b/app/jobs/cache/minutes_logged_job.rb index 18b4f90cd..390226ccb 100644 --- a/app/jobs/cache/minutes_logged_job.rb +++ b/app/jobs/cache/minutes_logged_job.rb @@ -4,6 +4,6 @@ class Cache::MinutesLoggedJob < Cache::ActivityJob private def calculate - Heartbeat.coding_only.with_valid_timestamps.where(time: 1.hour.ago..Time.current).duration_seconds / 60 + Clickhouse::Heartbeat.coding_only.with_valid_timestamps.where(time: 1.hour.ago..Time.current).duration_seconds / 60 end end diff --git a/app/jobs/dashboard_rollup_refresh_job.rb b/app/jobs/dashboard_rollup_refresh_job.rb deleted file mode 100644 index 48e6ad57d..000000000 --- a/app/jobs/dashboard_rollup_refresh_job.rb +++ /dev/null @@ -1,28 +0,0 @@ -class DashboardRollupRefreshJob < ApplicationJob - queue_as :default - - include GoodJob::ActiveJobExtensions::Concurrency - - good_job_control_concurrency_with( - total_limit: 1, key: -> { "dashboard_rollup_refresh_job_#{arguments.first}" } - ) - - DEFAULT_WAIT = 2.minutes - ENQUEUE_CACHE_KEY_PREFIX = "dashboard_rollup_refresh_enqueued".freeze - - def self.schedule_for(user_id, wait: DEFAULT_WAIT) - DashboardRollup.mark_dirty(user_id) - return unless Rails.cache.write(enqueue_cache_key(user_id), true, expires_in: wait + 1.minute, unless_exist: true) - set(wait: wait).perform_later(user_id) - end - - def self.enqueue_cache_key(user_id) = "#{ENQUEUE_CACHE_KEY_PREFIX}_#{user_id}" - - def perform(user_id) - user = User.find_by(id: user_id) - return unless user - DashboardRollupRefreshService.new(user:).call - ensure - Rails.cache.delete(self.class.enqueue_cache_key(user_id)) - end -end diff --git a/app/jobs/geocode_users_without_country_job.rb b/app/jobs/geocode_users_without_country_job.rb index 8fce59b88..818279439 100644 --- a/app/jobs/geocode_users_without_country_job.rb +++ b/app/jobs/geocode_users_without_country_job.rb @@ -7,21 +7,14 @@ class GeocodeUsersWithoutCountryJob < ApplicationJob enqueue_limit 1 def perform - rows = ActiveRecord::Base.connection.select_rows(<<~SQL.squish) - SELECT u.id AS user_id, h.ip_address - FROM ( - SELECT id FROM users WHERE country_code IS NULL - ) AS u(id) - CROSS JOIN LATERAL ( - SELECT ip_address - FROM heartbeats - WHERE heartbeats.user_id = u.id - AND heartbeats.ip_address IS NOT NULL - AND heartbeats.deleted_at IS NULL - ORDER BY heartbeats.id DESC - LIMIT 1 - ) AS h - SQL + user_ids = User.where(country_code: nil).pluck(:id) + return if user_ids.empty? + + # Latest (by insertion id) non-null IP per user, matching the old LATERAL ... LIMIT 1. + rows = Clickhouse::Heartbeat + .where(user_id: user_ids).where.not(ip_address: nil) + .group(:user_id) + .pluck(Arel.sql("user_id, argMax(ip_address, id)")) return if rows.empty? diff --git a/app/jobs/heartbeat_export_job.rb b/app/jobs/heartbeat_export_job.rb index 3abf9f81f..1b6b54d49 100644 --- a/app/jobs/heartbeat_export_job.rb +++ b/app/jobs/heartbeat_export_job.rb @@ -27,8 +27,8 @@ def perform(user_id, all_data:, start_date: nil, end_date: nil) end if all_data - heartbeats = user.heartbeats.order(time: :asc) - first_time, last_time = user.heartbeats.pick(Arel.sql("MIN(time), MAX(time)")) + heartbeats = Clickhouse::Heartbeat.for_user(user).order(time: :asc, id: :asc) + first_time, last_time = Clickhouse::Heartbeat.for_user(user).pick(Arel.sql("MIN(time), MAX(time)")) if first_time && last_time start_date = Time.at(first_time).to_date end_date = Time.at(last_time).to_date @@ -38,9 +38,9 @@ def perform(user_id, all_data:, start_date: nil, end_date: nil) else start_date = Date.iso8601(start_date) end_date = Date.iso8601(end_date) - heartbeats = user.heartbeats + heartbeats = Clickhouse::Heartbeat.for_user(user) .where("time >= ? AND time <= ?", start_date.beginning_of_day.to_f, end_date.end_of_day.to_f) - .order(time: :asc) + .order(time: :asc, id: :asc) end export_data = build_export_data(heartbeats, start_date, end_date) diff --git a/app/jobs/leaderboard_update_job.rb b/app/jobs/leaderboard_update_job.rb index cc2c2463e..82f6047c1 100644 --- a/app/jobs/leaderboard_update_job.rb +++ b/app/jobs/leaderboard_update_job.rb @@ -28,17 +28,18 @@ def build_leaderboard(date, period, force_update = false) eligible_users = User.where.not(github_uid: nil).where.not(trust_level: User.trust_levels[:red]) ActiveRecord::Base.transaction do - data = Heartbeat.where(user_id: eligible_users.select(:id), time: range) - .leaderboard_eligible - .group(:user_id).duration_seconds - .filter { |_, seconds| seconds > 60 } + data = Clickhouse::Heartbeat.where(time: range) + .leaderboard_eligible + .group(:user_id).duration_seconds + .filter { |_, seconds| seconds > 60 } + data = data.slice(*eligible_users.where(id: data.keys).pluck(:id)) # Two-phase streak: query 8d first (covers most), extend to 31d for users who maxed it. - streaks = Heartbeat.daily_streaks_for_users(data.keys, start_date: 8.days.ago, exclude_browser_time: true) + streaks = Clickhouse::Heartbeat.daily_streaks_for_users(data.keys, start_date: 8.days.ago, exclude_browser_time: true) needs_full_history = streaks.select { |_, s| s >= 6 }.keys if needs_full_history.any? needs_full_history.each { |id| Rails.cache.delete("user_streak_without_browser_v3_#{id}") } - streaks.merge!(Heartbeat.daily_streaks_for_users(needs_full_history, start_date: 31.days.ago, exclude_browser_time: true)) + streaks.merge!(Clickhouse::Heartbeat.daily_streaks_for_users(needs_full_history, start_date: 31.days.ago, exclude_browser_time: true)) end entries = data.map do |user_id, seconds| diff --git a/app/jobs/sailors_log_poll_for_changes_job.rb b/app/jobs/sailors_log_poll_for_changes_job.rb index 1350b2a41..5e74f35db 100644 --- a/app/jobs/sailors_log_poll_for_changes_job.rb +++ b/app/jobs/sailors_log_poll_for_changes_job.rb @@ -10,13 +10,18 @@ class SailorsLogPollForChangesJob < ApplicationJob IGNORED_PROJECTS = [ "<>", "Unknown" ].freeze def perform - users_who_coded = Heartbeat.with_valid_timestamps.where(time: 10.minutes.ago..).distinct.pluck(:user_id) + users_who_coded = Clickhouse::Heartbeat.with_valid_timestamps + .where(time: 10.minutes.ago.to_f..) + .distinct.pluck(:user_id) slack_uids = User.where(id: users_who_coded).pluck(:slack_uid) - new_notifs = SailorsLog.includes(:user, :notification_preferences) - .where(notification_preferences: { enabled: true }) - .where(slack_uid: slack_uids) - .flat_map { |sl| update_sailors_log(sl) } + sailors_logs = SailorsLog.includes(:user, :notification_preferences) + .where(notification_preferences: { enabled: true }) + .where(slack_uid: slack_uids) + .to_a + durations_by_user = Clickhouse::Heartbeat.project_durations_for_users(sailors_logs.map { |sl| sl.user.id }) + + new_notifs = sailors_logs.flat_map { |sl| update_sailors_log(sl, durations_by_user[sl.user.id] || {}) } notifs_to_send = SailorsLogSlackNotification.insert_all(new_notifs) notif_ids = notifs_to_send.result.to_a.map { |r| r["id"] } @@ -25,18 +30,9 @@ def perform private - def update_sailors_log(sailors_log) + def update_sailors_log(sailors_log, project_durations) return [] if sailors_log.user.active_remote_heartbeat_import_run? - project_durations = DashboardRollup - .where(user_id: sailors_log.user.id, dimension: "project", bucket_value_present: true) - .pluck(:bucket_value, :total_seconds).to_h - - if project_durations.empty? - DashboardRollupRefreshJob.schedule_for(sailors_log.user.id, wait: 0.seconds) - return [] - end - project_updates = [] project_durations.each do |k, v| next if ignored_project?(k) diff --git a/app/jobs/scan_github_repos_job.rb b/app/jobs/scan_github_repos_job.rb index 059b67667..a98f003ed 100644 --- a/app/jobs/scan_github_repos_job.rb +++ b/app/jobs/scan_github_repos_job.rb @@ -16,7 +16,7 @@ def perform(user_id = nil) scope.find_each(batch_size: 100) do |user| Rails.logger.info "Scanning GitHub repos for user #{user.id} (#{user.github_username})" existing = user.project_repo_mappings.pluck(:project_name) - project_names = user.heartbeats.where.not(project: existing).distinct.pluck(:project).compact + project_names = Clickhouse::Heartbeat.for_user(user).where.not(project: existing).distinct.pluck(:project).compact project_names.each do |project_name| Rails.cache.fetch("attempt_project_repo_mapping_job_#{user.id}_#{project_name}", expires_in: 1.hour) do diff --git a/app/jobs/sync_all_user_repo_events_job.rb b/app/jobs/sync_all_user_repo_events_job.rb index 9fd23be93..6324b7581 100644 --- a/app/jobs/sync_all_user_repo_events_job.rb +++ b/app/jobs/sync_all_user_repo_events_job.rb @@ -8,8 +8,9 @@ def perform Rails.logger.info "Kicking off SyncAllUserRepoEventsJob" # Users with GitHub auth that had heartbeats in the last 6 hours + recently_active_ids = Clickhouse::Heartbeat.where("created_at >= ?", 6.hours.ago).distinct.pluck(:user_id) users = User.where.not(github_access_token: nil).where.not(github_username: nil) - .joins(:heartbeats).where("heartbeats.created_at >= ?", 6.hours.ago).distinct + .where(id: recently_active_ids) if users.empty? Rails.logger.info "No users eligible for GitHub event sync at this time." diff --git a/app/jobs/weekly_summary_email_job.rb b/app/jobs/weekly_summary_email_job.rb index d4f7dbdec..e9752f2d6 100644 --- a/app/jobs/weekly_summary_email_job.rb +++ b/app/jobs/weekly_summary_email_job.rb @@ -16,17 +16,10 @@ def perform(reference_time = Time.current) def eligible_users(cutoff) users = User.arel_table - heartbeats = Heartbeat.arel_table - - recent_activity_exists = Heartbeat.unscoped - .where(heartbeats[:user_id].eq(users[:id])) - .where(heartbeats[:deleted_at].eq(nil)) - .where(heartbeats[:time].gteq(cutoff.to_f)) - .arel - .exists + recently_active_ids = Clickhouse::Heartbeat.where(time: cutoff.to_f..).distinct.pluck(:user_id) User.subscribed("weekly_summary").where( - users[:created_at].gteq(cutoff).or(recent_activity_exists) + users[:created_at].gteq(cutoff).or(users[:id].in(recently_active_ids)) ).where.not(id: DeletionRequest.active.select(:user_id)) end end diff --git a/app/mailers/weekly_summary_mailer.rb b/app/mailers/weekly_summary_mailer.rb index 87a3b50fb..bd8fe76f2 100644 --- a/app/mailers/weekly_summary_mailer.rb +++ b/app/mailers/weekly_summary_mailer.rb @@ -14,14 +14,14 @@ def weekly_summary(user, recipient_email:, starts_at:, ends_at:) @subject_period_label = "#{@starts_at_local.strftime("%b %-d")} - #{@ends_at_local.strftime("%b %-d, %Y")}" @period_label = @subject_period_label - coding_heartbeats = @user.heartbeats.where(time: @starts_at.to_f...@ends_at.to_f) - @total_seconds = coding_heartbeats.duration_seconds + coding_heartbeats = Clickhouse::Heartbeat.for_user(@user).where(time: @starts_at.to_f...@ends_at.to_f) + @total_seconds = Clickhouse::Heartbeat.duration_seconds(coding_heartbeats) num_days = [ (@ends_at - @starts_at) / 1.day, 1 ].max @daily_average_seconds = (@total_seconds / num_days).round @total_heartbeats = coding_heartbeats.count @active_days = active_days_count(coding_heartbeats) - @top_projects = breakdown(coding_heartbeats.group(:project).duration_seconds, default_name: "Other") - @top_languages = breakdown(Heartbeat.attributed_durations_by(coding_heartbeats, :language)) + @top_projects = breakdown(Clickhouse::Heartbeat.duration_seconds(coding_heartbeats.group(:project)), default_name: "Other") + @top_languages = breakdown(Clickhouse::Heartbeat.attributed_durations_by(coding_heartbeats, :language)) mail(to: recipient_email, subject: "Your Hackatime weekly summary (#{@subject_period_label})") end @@ -37,8 +37,12 @@ def breakdown(pairs, limit: 5, default_name: nil) end def active_days_count(scope) - timezone_sql = ActiveRecord::Base.connection.quote(@timezone_label) - scope.where.not(time: nil).distinct.count(Arel.sql("DATE(to_timestamp(time) AT TIME ZONE #{timezone_sql})")) + timezone_sql = Clickhouse::Heartbeat.connection.quote(@timezone_label) + relation_sql = scope.where.not(time: nil).select(:time).to_sql + Clickhouse::Heartbeat.connection.select_value(<<~SQL.squish).to_i + SELECT COUNT(DISTINCT toDate(toTimeZone(toDateTime64(time, 3), #{timezone_sql}))) + FROM (#{relation_sql}) weekly_summary_heartbeats + SQL rescue StandardError scope.where.not(time: nil).pluck(:time).map { |t| Time.at(t).in_time_zone(@timezone_label).to_date }.uniq.count end diff --git a/app/models/clickhouse/heartbeat.rb b/app/models/clickhouse/heartbeat.rb new file mode 100644 index 000000000..071487b0a --- /dev/null +++ b/app/models/clickhouse/heartbeat.rb @@ -0,0 +1,347 @@ +module Clickhouse + class Heartbeat < Clickhouse::Record + include TimeRangeFilterable + + self.table_name = "heartbeats" + self.inheritance_column = nil + # The gem infers a composite primary key from the ORDER BY tuple, which + # hijacks #id; the table carries the real Postgres id column. + self.primary_key = "id" + + BROWSER_EDITORS = %w[arc brave chrome chromium edge firefox floorp librewolf microsoft-edge opera opera-gx safari vivaldi waterfox zen].freeze + + enum :source_type, { + direct_entry: 0, + wakapi_import: 1, + test_entry: 2 + } + + # Keys ordered to match the historical Postgres attributes-hash order so + # fields_hash stays stable across the PG -> ClickHouse write cutover. + FIELDS_HASH_KEY_ORDER = %w[ + branch category cursorpos dependencies editor entity is_write language + line_additions line_deletions lineno lines machine operating_system + project project_root_count time type user_agent user_id + ].freeze + + INTEGER_HASH_KEYS = %w[cursorpos line_additions line_deletions lineno lines project_root_count user_id].freeze + STRING_HASH_KEYS = %w[branch category editor entity language machine operating_system project type user_agent].freeze + + default_scope { final.where(deleted_at: nil) } + + time_range_filterable_field :time + + scope :coding_only, -> { where(category: "coding") } + scope :excluding_browser_time, -> { where("editor IS NULL OR lower(editor) NOT IN (?)", BROWSER_EDITORS) } + scope :with_valid_timestamps, -> { where("time >= 0 AND time <= ?", 253402300799) } + scope :leaderboard_eligible, -> { coding_only.excluding_browser_time.with_valid_timestamps } + + scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) } + scope :recent, -> { where("time > ?", 24.hours.ago.to_i) } + scope :for_user, ->(user) { where(user_id: user.respond_to?(:id) ? user.id : user) } + scope :with_deleted, -> { unscope(where: :deleted_at) } + scope :only_deleted, -> { with_deleted.where.not(deleted_at: nil) } + + class << self + def heartbeat_timeout_duration(duration = nil) + duration ? (@heartbeat_timeout_duration = duration) : (@heartbeat_timeout_duration || 2.minutes) + end + + def indexed_attributes = FIELDS_HASH_KEY_ORDER + + def generate_fields_hash(attributes) + attrs = attributes.transform_keys(&:to_s) + normalized = FIELDS_HASH_KEY_ORDER.index_with { |key| cast_fields_hash_value(key, attrs[key]) } + Digest::MD5.hexdigest(normalized.to_json) + end + + def recent_count = Cache::HeartbeatCountsJob.perform_now[:recent_count] + def recent_imported_count = Cache::HeartbeatCountsJob.perform_now[:recent_imported_count] + + def safe_exists?(scope = all) + inner = scope.unscope(:select, :order).select(Arel.sql("1 AS one")).limit(1).to_sql + connection.select_all("SELECT one FROM (#{inner}) AS existence_check LIMIT 1").any? + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + false + end + + def duration_seconds(scope = all) + scope = scope.with_valid_timestamps + timeout = heartbeat_timeout_duration.to_i + + if scope.group_values.any? + raise NotImplementedError, "Multiple group values are not supported" if scope.group_values.length > 1 + + group_column = scope.group_values.first + group_expr = group_column.to_s.include?("(") ? group_column.to_s : connection.quote_column_name(group_column) + inner = deduped(scope).unscope(:group) + .select(Arel.sql("#{group_expr} AS grouped_time, #{capped_diff_sql(timeout, group_expr)} AS diff")) + .to_sql + + connection.select_all("SELECT grouped_time, SUM(diff) AS duration FROM (#{inner}) AS diffs GROUP BY grouped_time") + .each_with_object({}) { |row, hash| hash[row["grouped_time"]] = row["duration"].to_f.round } + else + inner = deduped(scope).select(Arel.sql("#{capped_diff_sql(timeout)} AS diff")).to_sql + connection.select_all("SELECT SUM(diff) AS duration FROM (#{inner}) AS diffs").first["duration"].to_f.round + end + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + scope.group_values.any? ? {} : 0 + end + + def duration_seconds_boundary_aware(scope, start_time, end_time, excluded_categories: []) + timeout = heartbeat_timeout_duration.to_i + start_f = to_epoch(start_time) + end_f = to_epoch(end_time) + + base = with_valid_timestamps + base = base.where("lower(category) NOT IN (?)", excluded_categories) if excluded_categories.present? + + where_values = scope.where_values_hash + %w[user_id category project].each do |key| + base = base.where(key => where_values[key]) if where_values[key] + end + + boundary_time = deduped(base).where("time < ?", start_f).order(time: :desc, id: :desc).limit(1).pick(:time) + + combined = if boundary_time + deduped(base).where("time >= ? OR time = ?", start_f, boundary_time).where("time <= ?", end_f) + else + deduped(base).where(time: start_f..end_f) + end + + inner = combined.select(Arel.sql("time, #{capped_diff_sql(timeout)} AS diff")).to_sql + connection.select_value("SELECT SUM(diff) FROM (#{inner}) AS diffs WHERE time >= #{start_f.to_f}").to_f.round + end + + def duration_formatted(scope = all) + seconds = duration_seconds(scope) + format("%02d:%02d:%02d", seconds / 3600, (seconds % 3600) / 60, seconds % 60) + end + + def duration_simple(scope = all) + seconds = duration_seconds(scope) + hours = seconds / 3600 + return "#{hours} hrs" if hours > 1 + return "1 hr" if hours == 1 + + "#{(seconds % 3600) / 60} min" + end + + def daily_durations(user_timezone:, start_date: 365.days.ago, end_date: Time.current) + timeout = heartbeat_timeout_duration.to_i + day_expr = day_group_sql(user_timezone) + + inner = deduped(all.with_valid_timestamps.where(time: start_date..end_date)) + .select(Arel.sql("#{day_expr} AS day_group, #{capped_diff_sql(timeout, day_expr)} AS diff")) + .to_sql + + connection.select_all("SELECT day_group, SUM(diff) AS duration FROM (#{inner}) AS diffs GROUP BY day_group ORDER BY day_group") + .map { |row| [ Date.parse(row["day_group"].to_s), row["duration"].to_f.round ] } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + [] + end + + def attributed_durations_by(scope, field) + scope = scope.with_valid_timestamps + timeout = heartbeat_timeout_duration.to_i + field_expr = connection.quote_column_name(field.to_s) + + inner = deduped(scope).unscope(:group, :select, :order) + .select(Arel.sql("#{field_expr} AS bucket, #{capped_diff_sql(timeout)} AS diff")) + .to_sql + + connection.select_all(<<~SQL.squish) + SELECT bucket, SUM(diff) AS duration + FROM (#{inner}) AS diffs + WHERE bucket IS NOT NULL AND bucket <> '' + GROUP BY bucket + SQL + .each_with_object({}) { |row, hash| hash[row["bucket"]] = row["duration"].to_f.round } + end + + def to_span(timeout_duration: nil) + timeout = (timeout_duration || heartbeat_timeout_duration.to_i).to_i + times = deduped(all.with_valid_timestamps).order(:time).pluck(:time) + return [] if times.empty? + + spans = [] + current_span_start = times.first + + times.each_with_index do |current_time, index| + next_time = times[index + 1] + next unless next_time.nil? || (next_time - current_time) > timeout + + base_duration = (current_time - current_span_start).round + if next_time + gap_duration = [ next_time - current_time, timeout ].min + total_duration = base_duration + gap_duration + end_time = current_time + gap_duration + else + total_duration = base_duration + end_time = current_time + end + + spans << { start_time: current_span_start, end_time: end_time, duration: total_duration } if total_duration > 0 + current_span_start = next_time if next_time + end + + spans + end + + def project_durations_for_users(user_ids) + return {} if user_ids.empty? + + timeout = heartbeat_timeout_duration.to_i + diff_sql = "least(" \ + "time - lagInFrame(time, 1, time) OVER (" \ + "PARTITION BY user_id, project ORDER BY time, id " \ + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " \ + "#{timeout})" + inner = deduped(with_valid_timestamps.where(user_id: user_ids)) + .select(Arel.sql("user_id, project, #{diff_sql} AS diff")) + .to_sql + + rows = connection.select_all(<<~SQL.squish) + SELECT user_id, project, SUM(diff) AS duration + FROM (#{inner}) AS diffs + GROUP BY user_id, project + SQL + + rows.each_with_object({}) do |row, result| + (result[row["user_id"].to_i] ||= {})[row["project"]] = row["duration"].to_f.round + end + end + + def daily_streaks_for_users(user_ids, start_date: 31.days.ago, exclude_browser_time: false) + return {} if user_ids.empty? + start_date = [ start_date, 31.days.ago ].max + cache_prefix = exclude_browser_time ? "user_streak_without_browser_v3" : "user_streak_v3" + streak_cache = Rails.cache.read_multi(*user_ids.map { |id| "#{cache_prefix}_#{id}" }) + + uncached_users = user_ids.select { |id| streak_cache["#{cache_prefix}_#{id}"].nil? } + return user_ids.index_with { |id| streak_cache["#{cache_prefix}_#{id}"] || 0 } if uncached_users.empty? + + timeout = heartbeat_timeout_duration.to_i + tz_by_user = ::User.where(id: uncached_users).pluck(:id, :timezone).to_h + users_by_tz = uncached_users.group_by { |id| resolve_timezone(tz_by_user[id]) } + + daily_durations = {} + users_by_tz.each do |timezone, ids| + day_expr = day_group_sql(timezone) + scope = with_valid_timestamps + .where(user_id: ids) + .where.not(category: "browsing") + .where(time: start_date..Time.current) + scope = scope.excluding_browser_time if exclude_browser_time + + partition = "user_id, #{day_expr}" + # PG quirk preserved: LEAST(NULL, timeout) = timeout, so the first row + # of each (user, day) contributes the full timeout, not 0. + diff_sql = "least(" \ + "time - lagInFrame(time, 1, time - #{timeout}) OVER (" \ + "PARTITION BY #{partition} ORDER BY time, id " \ + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " \ + "#{timeout})" + inner = deduped(scope) + .select(Arel.sql("user_id, #{day_expr} AS day_group, #{diff_sql} AS diff")) + .to_sql + + rows = connection.select_all(<<~SQL.squish) + SELECT user_id, day_group, SUM(diff) AS duration + FROM (#{inner}) AS diffs + GROUP BY user_id, day_group + SQL + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + rows = [] + ensure + rows ||= [] + + current_date = Time.current.in_time_zone(timezone).to_date + rows.group_by { |row| row["user_id"].to_i }.each do |user_id, user_rows| + daily_durations[user_id] = { + current_date: current_date, + days: user_rows.map { |row| [ Date.parse(row["day_group"].to_s), row["duration"].to_f.round ] } + .sort_by { |date, _| date }.reverse + } + end + end + + result = user_ids.index_with { |id| streak_cache["#{cache_prefix}_#{id}"] || 0 } + daily_durations.each do |user_id, data| + current_date = data[:current_date] + eligible_days = data[:days].filter_map { |date, duration| date if date <= current_date && duration >= 15 * 60 } + + streak = 0 + expected_date = eligible_days.first == current_date ? current_date : current_date - 1.day + eligible_days.each do |date| + if date == expected_date + streak += 1 + expected_date -= 1.day + elsif date < expected_date + break + end + end + + result[user_id] = streak + Rails.cache.write("#{cache_prefix}_#{user_id}", streak, expires_in: 1.hour) + end + + result + end + + private + + def deduped(scope) = scope.final + + def capped_diff_sql(timeout, partition_expr = nil) + partition = partition_expr ? "PARTITION BY #{partition_expr} " : "" + "least(" \ + "time - lagInFrame(time, 1, time) OVER (" \ + "#{partition}ORDER BY time, id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), " \ + "#{timeout})" + end + + def day_group_sql(timezone) + "toDate(toTimeZone(toDateTime64(time, 3), #{connection.quote(resolve_timezone(timezone))}))" + end + + def resolve_timezone(timezone) + return timezone if TZInfo::Timezone.all_identifiers.include?(timezone) + + Rails.logger.warn "Invalid timezone for ClickHouse duration query: #{timezone.inspect}. Defaulting to UTC." + "UTC" + end + + def to_epoch(value) + value.respond_to?(:to_f) ? value.to_f : value + end + + def cast_fields_hash_value(key, value) + return nil if value.nil? + + case key + when *INTEGER_HASH_KEYS then value.to_s.strip.empty? ? nil : value.to_i + when *STRING_HASH_KEYS then value.to_s + when "time" then value.to_f + when "is_write" then cast_boolean(value) + when "dependencies" then Array(value).map(&:to_s) + else value + end + end + + def cast_boolean(value) + return value if value == true || value == false + + %w[true t 1 yes on].include?(value.to_s.downcase) + end + end + end +end diff --git a/app/models/clickhouse/heartbeat_writer.rb b/app/models/clickhouse/heartbeat_writer.rb new file mode 100644 index 000000000..c500a47e2 --- /dev/null +++ b/app/models/clickhouse/heartbeat_writer.rb @@ -0,0 +1,134 @@ +module Clickhouse + # The only write path for heartbeats. Heartbeats live exclusively in + # ClickHouse; nothing writes them to Postgres anymore. + # + # Dedup model: rows sharing the ReplacingMergeTree ORDER BY key + # (user_id, time, fields_hash) collapse eventually, keeping the highest + # version. Soft deletes and merges are new row versions, never mutations. + module HeartbeatWriter + WRITABLE_COLUMNS = %w[ + id user_id time fields_hash project branch entity category editor + language machine operating_system type user_agent ip_address dependencies + lineno lines cursorpos line_additions line_deletions project_root_count + is_write source_type ja4_id deleted_at created_at updated_at version + ].freeze + + class << self + # JS-safe snowflake: (epoch seconds << 22) | 22 random bits stays under + # 2**53 until 2038. Collisions are harmless: id is not a dedup key, only + # an attribution tie-breaker. + def generate_id(time = Time.current) + (time.to_i << 22) | SecureRandom.random_number(1 << 22) + end + + def create!(attrs) + rows = insert_rows([ attrs ]) + rows.first + end + + def insert_rows(attribute_hashes) + rows = attribute_hashes.map { |attributes| shape_row(attributes) } + if rows.any? + Clickhouse::Heartbeat.unscoped.insert_all(rows) + clear_query_cache + end + rows + end + + # Server-side bulk soft delete: re-insert every live row for the user + # with deleted_at set and a fresh version. + def soft_delete_user_heartbeats!(user_id) + connection = Clickhouse::Heartbeat.connection + table = connection.quote_table_name(Clickhouse::Heartbeat.table_name) + version = (Time.current.to_f * 1_000_000).round + + exprs = WRITABLE_COLUMNS.map do |column| + case column + when "deleted_at" then "now64(6)" + when "version" then version.to_s + else connection.quote_column_name(column) + end + end + connection.execute(<<~SQL.squish) + INSERT INTO #{table} (#{column_list(connection)}) + SELECT #{exprs.join(", ")} FROM #{table} FINAL + WHERE user_id = #{user_id.to_i} AND deleted_at IS NULL + SQL + clear_query_cache + end + + # Account merge: copy the newer user's live rows to the older user, then + # tombstone everything left under the newer user. Idempotent — re-running + # re-inserts identical rows which ReplacingMergeTree collapses. + def merge_user_heartbeats!(older_user_id:, newer_user_id:) + connection = Clickhouse::Heartbeat.connection + table = connection.quote_table_name(Clickhouse::Heartbeat.table_name) + version = (Time.current.to_f * 1_000_000).round + + moved_exprs = WRITABLE_COLUMNS.map do |column| + column == "user_id" ? older_user_id.to_i.to_s : connection.quote_column_name(column) + end + connection.execute(<<~SQL.squish) + INSERT INTO #{table} (#{column_list(connection)}) + SELECT #{moved_exprs.join(", ")} FROM #{table} FINAL + WHERE user_id = #{newer_user_id.to_i} AND deleted_at IS NULL + SQL + + tombstone_exprs = WRITABLE_COLUMNS.map do |column| + case column + when "deleted_at" then "now64(6)" + when "version" then version.to_s + else connection.quote_column_name(column) + end + end + connection.execute(<<~SQL.squish) + INSERT INTO #{table} (#{column_list(connection)}) + SELECT #{tombstone_exprs.join(", ")} FROM #{table} FINAL + WHERE user_id = #{newer_user_id.to_i} AND deleted_at IS NULL + SQL + clear_query_cache + end + + private + + def column_list(connection) + WRITABLE_COLUMNS.map { |column| connection.quote_column_name(column) }.join(", ") + end + + # The ClickHouse adapter's insert paths don't invalidate Rails' query + # cache, so reads issued after a write inside the same executor frame + # (request/job/test) would return stale results. + def clear_query_cache + Clickhouse::Heartbeat.connection.clear_query_cache + end + + def shape_row(attributes) + attrs = attributes.transform_keys(&:to_s) + now = Time.current + created_at = attrs["created_at"] || now + updated_at = attrs["updated_at"] || created_at + deleted_at = attrs["deleted_at"] + time = attrs["time"]&.to_f + + row = WRITABLE_COLUMNS.index_with { |column| attrs[column] } + row["id"] = attrs["id"] || generate_id(now) + row["time"] = time + row["fields_hash"] = attrs["fields_hash"].presence || Clickhouse::Heartbeat.generate_fields_hash(attrs) + row["ip_address"] = attrs["ip_address"]&.to_s + row["dependencies"] = Array(attrs["dependencies"]).map(&:to_s) + row["source_type"] = shape_source_type(attrs["source_type"]) + row["is_write"] = attrs["is_write"].nil? ? nil : Clickhouse::Heartbeat.send(:cast_boolean, attrs["is_write"]) + row["created_at"] = created_at + row["updated_at"] = updated_at + row["deleted_at"] = deleted_at + version_time = [ updated_at, deleted_at ].compact.max + row["version"] = attrs["version"] || (version_time.to_f * 1_000_000).round + row + end + + def shape_source_type(value) + value.presence || "direct_entry" + end + end + end +end diff --git a/app/models/clickhouse/record.rb b/app/models/clickhouse/record.rb new file mode 100644 index 000000000..fec263485 --- /dev/null +++ b/app/models/clickhouse/record.rb @@ -0,0 +1,7 @@ +module Clickhouse + class Record < ActiveRecord::Base + self.abstract_class = true + + establish_connection :clickhouse + end +end diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb deleted file mode 100644 index 744bf6001..000000000 --- a/app/models/concerns/heartbeatable.rb +++ /dev/null @@ -1,264 +0,0 @@ -module Heartbeatable - extend ActiveSupport::Concern - - BROWSER_EDITORS = %w[arc brave chrome chromium edge firefox floorp librewolf microsoft-edge opera opera-gx safari vivaldi waterfox zen].freeze - - included do - # Filter heartbeats to only include those with category equal to "coding" - scope :coding_only, -> { where(category: "coding") } - scope :excluding_browser_time, -> { - where("editor IS NULL OR LOWER(editor) NOT IN (?)", BROWSER_EDITORS) - } - scope :leaderboard_eligible, -> { coding_only.excluding_browser_time.with_valid_timestamps } - - # This is to prevent PG timestamp overflow errors if someones gives us a - # heartbeat with a time that is enormously far in the future. - scope :with_valid_timestamps, -> { where("time >= 0 AND time <= ?", 253402300799) } - end - - class_methods do - def heartbeat_timeout_duration(duration = nil) - duration ? (@heartbeat_timeout_duration = duration) : (@heartbeat_timeout_duration || 2.minutes) - end - - def to_span(timeout_duration: nil) - timeout_duration ||= heartbeat_timeout_duration.to_i - - heartbeats = with_valid_timestamps.order(time: :asc, id: :asc) - return [] if heartbeats.empty? - - sql = <<~SQL - SELECT - time, - LEAD(time) OVER (ORDER BY time, id) as next_time - FROM (#{heartbeats.to_sql}) AS heartbeats - SQL - - results = connection.select_all(sql) - return [] if results.empty? - - spans = [] - current_span_start = results.first["time"] - - results.each do |row| - current_time = row["time"] - next_time = row["next_time"] - - if next_time.nil? || (next_time - current_time) > timeout_duration - base_duration = (current_time - current_span_start).round - - if next_time - gap_duration = [ next_time - current_time, timeout_duration ].min - total_duration = base_duration + gap_duration - end_time = current_time + gap_duration - else - total_duration = base_duration - end_time = current_time - end - - if total_duration > 0 - spans << { - start_time: current_span_start, - end_time: end_time, - duration: total_duration - } - end - - current_span_start = next_time if next_time - end - end - - spans - end - - def duration_formatted(scope = all) - seconds = duration_seconds(scope) - format("%02d:%02d:%02d", seconds / 3600, (seconds % 3600) / 60, seconds % 60) - end - - def duration_simple(scope = all) - # 3 hours 10 min => "3 hrs" / 1 hour 10 min => "1 hr" / 10 min => "10 min" - seconds = duration_seconds(scope) - hours = seconds / 3600 - return "#{hours} hrs" if hours > 1 - return "1 hr" if hours == 1 - "#{(seconds % 3600) / 60} min" # 0 min if minutes is 0 - end - - def daily_streaks_for_users(user_ids, start_date: 31.days.ago, exclude_browser_time: false) - return {} if user_ids.empty? - start_date = [ start_date, 31.days.ago ].max - cache_prefix = exclude_browser_time ? "user_streak_without_browser_v3" : "user_streak_v3" - streak_cache = Rails.cache.read_multi(*user_ids.map { |id| "#{cache_prefix}_#{id}" }) - - uncached_users = user_ids.select { |id| streak_cache["#{cache_prefix}_#{id}"].nil? } - return user_ids.index_with { |id| streak_cache["#{cache_prefix}_#{id}"] || 0 } if uncached_users.empty? - - timeout = heartbeat_timeout_duration.to_i - day_group_sql = "DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone)" - streak_diff_sql = <<~SQL.squish - LEAST( - time - LAG(time) OVER (PARTITION BY user_id, #{day_group_sql} ORDER BY time, #{quoted_table_name}.id), - #{timeout} - ) as diff - SQL - raw_durations = joins(:user) - .where(user_id: uncached_users) - .where.not(category: "browsing") - .with_valid_timestamps - .where(time: start_date..Time.current) - .select( - :user_id, - "users.timezone as user_timezone", - Arel.sql("#{day_group_sql} as day_group"), - Arel.sql(streak_diff_sql) - ) - raw_durations = raw_durations.excluding_browser_time if exclude_browser_time - - # Then aggregate the results - daily_durations = connection.select_all( - "SELECT user_id, user_timezone, day_group, COALESCE(SUM(diff), 0)::integer as duration - FROM (#{raw_durations.to_sql}) AS diffs - GROUP BY user_id, user_timezone, day_group" - ).group_by { |row| row["user_id"] } - .transform_values do |rows| - timezone = rows.first["user_timezone"] - - if timezone.blank? - Rails.logger.warn "nil tz, going to utc." - timezone = "UTC" - else - begin - TZInfo::Timezone.get(timezone) - rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError - Rails.logger.warn "Invalid timezone for streak calculation: #{timezone}. Defaulting to UTC." - timezone = "UTC" - end - end - - current_date = Time.current.in_time_zone(timezone).to_date - { - current_date: current_date, - days: rows.map do |row| - [ row["day_group"].to_date, row["duration"].to_i ] - end.sort_by { |date, _| date }.reverse - } - end - - result = user_ids.index_with { |id| streak_cache["#{cache_prefix}_#{id}"] || 0 } - - # Then calculate streaks for each user - daily_durations.each do |user_id, data| - current_date = data[:current_date] - days = data[:days] - - eligible_days = days.filter_map do |date, duration| - date if date <= current_date && duration >= 15 * 60 - end - - streak = 0 - expected_date = eligible_days.first == current_date ? current_date : current_date - 1.day - - eligible_days.each do |date| - if date == expected_date - streak += 1 - expected_date -= 1.day - elsif date < expected_date - break - end - end - - result[user_id] = streak - - # Cache the streak for 1 hour - Rails.cache.write("#{cache_prefix}_#{user_id}", streak, expires_in: 1.hour) - end - - result - end - - def daily_durations(user_timezone:, start_date: 365.days.ago, end_date: Time.current) - timezone = user_timezone - unless TZInfo::Timezone.all_identifiers.include?(timezone) - Rails.logger.warn "Invalid timezone provided to daily_durations: #{timezone}. Defaulting to UTC." - timezone = "UTC" - end - - day_trunc = Arel.sql("DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE '#{timezone}')") - select(day_trunc.as("day_group")).where(time: start_date..end_date).group(day_trunc).duration_seconds - .map { |date, duration| [ date.to_date, duration ] } - end - - def attributed_durations_by(scope, field) - scope = scope.with_valid_timestamps - timeout = heartbeat_timeout_duration.to_i - field_expr = connection.quote_column_name(field.to_s) - base_sql = scope.unscope(:group, :select, :order).select(:id, :time, field).to_sql - - sql = <<~SQL.squish - SELECT bucket, COALESCE(SUM(diff), 0)::integer AS duration - FROM ( - SELECT #{field_expr} AS bucket, - CASE WHEN LAG(time) OVER (ORDER BY time, id) IS NULL THEN 0 - ELSE LEAST(time - LAG(time) OVER (ORDER BY time, id), #{timeout}) END AS diff - FROM (#{base_sql}) heartbeats_for_attribution - ) capped_diffs - WHERE bucket IS NOT NULL AND bucket <> '' - GROUP BY bucket - SQL - - connection.select_all(sql).each_with_object({}) { |row, hash| hash[row["bucket"]] = row["duration"].to_i } - end - - def duration_seconds(scope = all) - scope = scope.with_valid_timestamps - timeout = heartbeat_timeout_duration.to_i - - if scope.group_values.any? - raise NotImplementedError, "Multiple group values are not supported" if scope.group_values.length > 1 - - group_column = scope.group_values.first - # Don't quote if it's a SQL function (contains parentheses) - group_expr = group_column.to_s.include?("(") ? group_column : connection.quote_column_name(group_column) - - capped_diffs = scope.select("#{group_expr} as grouped_time, CASE WHEN LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time, #{quoted_table_name}.id) IS NULL THEN 0 ELSE LEAST(time - LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time, #{quoted_table_name}.id), #{timeout}) END as diff") - .where.not(time: nil).unscope(:group) - - connection.select_all( - "SELECT grouped_time, COALESCE(SUM(diff), 0)::integer as duration FROM (#{capped_diffs.to_sql}) AS diffs GROUP BY grouped_time" - ).each_with_object({}) { |row, hash| hash[row["grouped_time"]] = row["duration"].to_i } - else - # when not grouped, return a single value - capped_diffs = scope.select("CASE WHEN LAG(time) OVER (ORDER BY time, #{quoted_table_name}.id) IS NULL THEN 0 ELSE LEAST(time - LAG(time) OVER (ORDER BY time, #{quoted_table_name}.id), #{timeout}) END as diff").where.not(time: nil) - connection.select_value("SELECT COALESCE(SUM(diff), 0)::integer FROM (#{capped_diffs.to_sql}) AS diffs").to_i - end - end - - def duration_seconds_boundary_aware(scope, start_time, end_time, excluded_categories: []) - scope = scope.with_valid_timestamps - base_scope = scope.model.all.with_valid_timestamps - base_scope = base_scope.where.not("LOWER(category) IN (?)", excluded_categories) if excluded_categories.present? - - where_values = scope.where_values_hash - %w[user_id category project deleted_at].each do |key| - base_scope = base_scope.where(key => where_values[key]) if where_values[key] - end - - # get the heartbeat before the start_time - boundary_heartbeat = base_scope.where("time < ?", start_time).order(time: :desc, id: :desc).limit(1).first - - # if it's not NULL, we'll use it - combined_scope = boundary_heartbeat ? - base_scope.where("time >= ? OR time = ?", start_time, boundary_heartbeat.time).where("time <= ?", end_time) : - base_scope.where(time: start_time..end_time) - - # we calc w/ the boundary heartbeat, but we only sum within the orignal constraint - timeout = heartbeat_timeout_duration.to_i - capped_diffs = combined_scope - .select("time, CASE WHEN LAG(time) OVER (ORDER BY time, #{quoted_table_name}.id) IS NULL THEN 0 ELSE LEAST(time - LAG(time) OVER (ORDER BY time, #{quoted_table_name}.id), #{timeout}) END as diff") - .where.not(time: nil).order(time: :asc, id: :asc) - - connection.select_value("SELECT COALESCE(SUM(diff), 0)::integer FROM (#{capped_diffs.to_sql}) AS diffs WHERE time >= #{connection.quote(start_time)}").to_i - end - end -end diff --git a/app/models/concerns/slack_integration.rb b/app/models/concerns/slack_integration.rb index ff907a747..94d30d405 100644 --- a/app/models/concerns/slack_integration.rb +++ b/app/models/concerns/slack_integration.rb @@ -56,11 +56,12 @@ def update_slack_status return if status_present && status_custom - current_project = heartbeats.order(time: :desc).first&.project + current_project = Clickhouse::Heartbeat.for_user(self).order(time: :desc, id: :desc).first&.project current_project_duration = Time.use_zone(timezone) do - heartbeats.where(project: current_project) - .today - .duration_seconds + Clickhouse::Heartbeat.for_user(self) + .where(project: current_project) + .today + .duration_seconds end current_project_duration_formatted = ApplicationController.helpers.short_time_simple(current_project_duration) diff --git a/app/models/dashboard_rollup.rb b/app/models/dashboard_rollup.rb deleted file mode 100644 index 233ddaa9d..000000000 --- a/app/models/dashboard_rollup.rb +++ /dev/null @@ -1,26 +0,0 @@ -class DashboardRollup < ApplicationRecord - DIMENSIONS = %w[total project project_details language editor operating_system category weekly_project activity_graph today_stats filter_options].freeze - TOTAL_DIMENSION = "total".freeze - PROJECT_DETAILS_DIMENSION = "project_details".freeze - ACTIVITY_GRAPH_DIMENSION = "activity_graph".freeze - TODAY_STATS_DIMENSION = "today_stats".freeze - FILTER_OPTIONS_DIMENSION = "filter_options".freeze - DIRTY_CACHE_KEY_PREFIX = "dashboard_rollup_dirty".freeze - - belongs_to :user - - validates :dimension, presence: true, inclusion: { in: DIMENSIONS } - validates :total_seconds, numericality: { greater_than_or_equal_to: 0 } - validates :bucket_value_present, inclusion: { in: [ true, false ] } - validates :source_heartbeats_count, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true - - scope :for_dimension, ->(dimension) { where(dimension: dimension.to_s) } - - def total_dimension? = dimension == TOTAL_DIMENSION - def bucket = bucket_value_present ? bucket_value : nil - - def self.dirty_cache_key(user_id) = "#{DIRTY_CACHE_KEY_PREFIX}_#{user_id}" - def self.mark_dirty(user_id) = Rails.cache.write(dirty_cache_key(user_id), true, expires_in: 1.day, unless_exist: true) - def self.clear_dirty(user_id) = Rails.cache.delete(dirty_cache_key(user_id)) - def self.dirty?(user_id) = Rails.cache.exist?(dirty_cache_key(user_id)) -end diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb deleted file mode 100644 index f638f2c4f..000000000 --- a/app/models/heartbeat.rb +++ /dev/null @@ -1,73 +0,0 @@ -class Heartbeat < ApplicationRecord - before_save :set_fields_hash! - before_save :set_time_epoch! - after_commit :schedule_dashboard_rollup_refresh, on: %i[create update destroy] - - include Heartbeatable - include TimeRangeFilterable - - time_range_filterable_field :time - - # Default scope to exclude deleted records - default_scope { where(deleted_at: nil) } - - scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) } - scope :recent, -> { where("time > ?", 24.hours.ago.to_i) } - scope :with_deleted, -> { unscope(where: :deleted_at) } - scope :only_deleted, -> { with_deleted.where.not(deleted_at: nil) } - - enum :source_type, { - direct_entry: 0, - wakapi_import: 1, - test_entry: 2 - } - - # This is to prevent Rails from trying to use STI even though we have a "type" column - self.inheritance_column = nil - - self.ignored_columns += %w[ysws_program] # unused - - belongs_to :user - belongs_to :ja4, optional: true - - validates :time, presence: true - - # after_create :mirror_to_wakatime - - def self.recent_count = Cache::HeartbeatCountsJob.perform_now[:recent_count] - def self.recent_imported_count = Cache::HeartbeatCountsJob.perform_now[:recent_imported_count] - - def self.generate_fields_hash(attributes) - Digest::MD5.hexdigest(attributes.transform_keys(&:to_s).slice(*self.indexed_attributes).to_json) - end - - def self.indexed_attributes - %w[user_id branch category dependencies editor entity language machine operating_system project type user_agent line_additions line_deletions lineno lines cursorpos project_root_count time is_write] - end - - def soft_delete - update_column(:deleted_at, Time.current) - DashboardRollupRefreshJob.schedule_for(user_id) - end - - def restore - update_column(:deleted_at, nil) - DashboardRollupRefreshJob.schedule_for(user_id) - end - - private - - def set_fields_hash! - # only if the field exists in activerecord - self.fields_hash = self.class.generate_fields_hash(self.attributes) if self.class.column_names.include?("fields_hash") - end - - # Populate the hypertable partition column on AR-object saves. Bulk ingest - # (insert/insert_all) sets it directly; this covers create/save/update paths. - # No-op until the column exists (post-cutover; plain table in dev/test). - def set_time_epoch! - self.time_epoch = time&.floor if self.class.column_names.include?("time_epoch") && time.present? - end - - def schedule_dashboard_rollup_refresh = DashboardRollupRefreshJob.schedule_for(user_id) -end diff --git a/app/models/ja4.rb b/app/models/ja4.rb index e4826e1a8..fda326f39 100644 --- a/app/models/ja4.rb +++ b/app/models/ja4.rb @@ -1,6 +1,4 @@ class Ja4 < ApplicationRecord - has_many :heartbeats, dependent: :nullify - validates :fingerprint, presence: true def self.resolve(fingerprint) diff --git a/app/models/sailors_log.rb b/app/models/sailors_log.rb index 62aba7187..8a9a869a3 100644 --- a/app/models/sailors_log.rb +++ b/app/models/sailors_log.rb @@ -16,7 +16,7 @@ class SailorsLog < ApplicationRecord def initialize_projects_summary return if projects_summary.present? - self.projects_summary = Heartbeat.where(user_id: user.id).group(:project).duration_seconds + self.projects_summary = Clickhouse::Heartbeat.for_user(user).group(:project).duration_seconds self.projects_summary ||= {} end diff --git a/app/models/sailors_log_leaderboard.rb b/app/models/sailors_log_leaderboard.rb index b58e0a421..796c722b7 100644 --- a/app/models/sailors_log_leaderboard.rb +++ b/app/models/sailors_log_leaderboard.rb @@ -45,13 +45,13 @@ def self.language_emoji(language) def self.generate_leaderboard_stats(channel) slack_ids_in_channel = SailorsLogNotificationPreference.where(enabled: true, slack_channel_id: channel) .distinct.pluck(:slack_uid) - users_in_channel = User.where(slack_uid: slack_ids_in_channel) - user_durations = Heartbeat.where(user: users_in_channel).today.group(:user_id).duration_seconds + user_ids_in_channel = User.where(slack_uid: slack_ids_in_channel).pluck(:id) + user_durations = Clickhouse::Heartbeat.where(user_id: user_ids_in_channel).today.group(:user_id).duration_seconds top_user_ids = user_durations.sort_by { |_, duration| -duration }.first(10).map(&:first) users_by_id = User.where(id: top_user_ids).index_by(&:id) top_user_ids.map do |user_id| - user_heartbeats = Heartbeat.where(user_id: user_id).today + user_heartbeats = Clickhouse::Heartbeat.for_user(user_id).today most_common_languages = user_heartbeats.where.not(language: nil).group(:project, :language).count .group_by { |k, _| k[0] } .transform_values { |langs| langs.max_by { |_, count| count }&.first&.last } diff --git a/app/models/user.rb b/app/models/user.rb index 253413025..f962c39b4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -183,7 +183,6 @@ def set_leaderboard_shadowban(banned:, changed_by_user:, reason: nil, expires_at false end - has_many :heartbeats has_many :goals, dependent: :destroy has_many :email_addresses, dependent: :destroy has_many :email_verification_requests, dependent: :destroy @@ -254,7 +253,7 @@ def schedule_leaderboard_shadowban_expiration foreign_key: :resource_owner_id, dependent: :delete_all - def streak_days = @streak_days ||= heartbeats.daily_streaks_for_users([ id ]).values.first + def streak_days = @streak_days ||= Clickhouse::Heartbeat.daily_streaks_for_users([ id ]).values.first def active_deletion_request = deletion_requests.active.order(created_at: :desc).first def pending_deletion? = active_deletion_request.present? def api_access_restricted? = red? || pending_deletion? @@ -283,7 +282,6 @@ def streak_days_formatted } after_update_commit :invalidate_activity_graph_cache, if: :saved_change_to_timezone? - after_update_commit :schedule_dashboard_rollup_refresh, if: :saved_change_to_timezone? def flipper_id = "User;#{id}" def active_remote_heartbeat_import_run? = heartbeat_import_runs.remote_imports.active_imports.exists? @@ -339,7 +337,12 @@ def display_name email.split("@")&.first.truncate(10) + " (email sign-up)" end - def most_recent_direct_entry_heartbeat = heartbeats.where(source_type: :direct_entry).order(time: :desc).first + def most_recent_direct_entry_heartbeat + Clickhouse::Heartbeat.for_user(self) + .where(source_type: :direct_entry) + .order(time: :desc, id: :desc) + .first + end def create_email_signin_token(continue_param: nil) = sign_in_tokens.create!(auth_type: :email, continue_param: continue_param) @@ -395,7 +398,6 @@ def invalidate_activity_graph_cache end end - def schedule_dashboard_rollup_refresh = DashboardRollupRefreshJob.schedule_for(id, wait: 0.seconds) def subscribe_to_default_lists = subscribe("weekly_summary") def schedule_welcome_email recipient_email = email_addresses.order(:id).pick(:email) diff --git a/app/services/anonymize_user_service.rb b/app/services/anonymize_user_service.rb index 31a56a5b1..0c36a20bc 100644 --- a/app/services/anonymize_user_service.rb +++ b/app/services/anonymize_user_service.rb @@ -45,7 +45,7 @@ def destroy_associated_records user.heartbeat_import_runs.destroy_all user.project_repo_mappings.destroy_all user.goals.destroy_all - Heartbeat.unscoped.where(user_id: user.id, deleted_at: nil).update_all(deleted_at: Time.current) + Clickhouse::HeartbeatWriter.soft_delete_user_heartbeats!(user.id) user.access_grants.destroy_all user.access_tokens.destroy_all end diff --git a/app/services/dashboard_data/snapshots.rb b/app/services/dashboard_data/snapshots.rb index 7c2f4b12d..5fbb9d92b 100644 --- a/app/services/dashboard_data/snapshots.rb +++ b/app/services/dashboard_data/snapshots.rb @@ -7,7 +7,7 @@ module Snapshots def grouped_durations_snapshot(scope) GROUPED_DIMENSIONS.index_with do |field| - field == :project ? project_grouped_durations(scope) : Heartbeat.attributed_durations_by(scope, field) + field == :project ? project_grouped_durations(scope) : scope.klass.attributed_durations_by(scope, field) end end @@ -22,50 +22,41 @@ def project_grouped_durations(scope) end def project_details_snapshot(scope:) - timeout = Heartbeat.heartbeat_timeout_duration.to_i + timeout = Clickhouse::Heartbeat.heartbeat_timeout_duration.to_i relation_sql = scope.with_valid_timestamps - .where.not(project: [ nil, "" ], time: nil) + .where.not(project: [ nil, "" ]) + .where.not(time: nil) .select(:id, :time, :project, :language) .to_sql - rows = Heartbeat.connection.select_all(<<~SQL.squish) + rows = Clickhouse::Heartbeat.connection.select_all(<<~SQL.squish) SELECT grouped_time, - COUNT(*)::integer AS heartbeat_count, + COUNT(*) AS heartbeat_count, MIN(time) AS first_heartbeat, MAX(time) AS last_heartbeat, - ARRAY_REMOVE(ARRAY_AGG(DISTINCT NULLIF(language, '')), NULL) AS languages, - COALESCE(SUM(diff), 0)::integer AS duration + groupUniqArrayIf(assumeNotNull(language), language IS NOT NULL AND language != '') AS languages, + SUM(diff) AS duration FROM ( SELECT project AS grouped_time, time, language, - CASE - WHEN LAG(time) OVER (PARTITION BY project ORDER BY time, id) IS NULL THEN 0 - ELSE LEAST(time - LAG(time) OVER (PARTITION BY project ORDER BY time, id), #{timeout}) - END AS diff - FROM (#{relation_sql}) project_detail_heartbeats - ) diffs + least(time - lagInFrame(time, 1, time) OVER (PARTITION BY project ORDER BY time, id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), #{timeout}) AS diff + FROM (#{relation_sql}) AS project_detail_heartbeats + ) AS diffs GROUP BY grouped_time SQL rows.each_with_object({}) do |row, result| result[row["grouped_time"]] = { - total_seconds: row["duration"].to_i, + total_seconds: row["duration"].to_f.round, total_heartbeats: row["heartbeat_count"].to_i, first_heartbeat: row["first_heartbeat"], last_heartbeat: row["last_heartbeat"], - languages: pg_array(row["languages"]).compact_blank + languages: Array(row["languages"]).compact_blank } end end - def pg_array(value) - return value if value.is_a?(Array) - return [] if value.blank? - - PG::TextDecoder::Array.new.decode(value.to_s) - end - def weekly_project_stats(user:, scope:) ranges = week_ranges(user.timezone) result = ranges.to_h { |week_key, *_| [ week_key, {} ] } @@ -76,31 +67,26 @@ def weekly_project_stats(user:, scope:) .select(:id, :time, :project) .to_sql - quoted_timezone = Heartbeat.connection.quote(user.timezone) - week_group_sql = "DATE_TRUNC('week', to_timestamp(time) AT TIME ZONE #{quoted_timezone})" + quoted_timezone = Clickhouse::Heartbeat.connection.quote(user.timezone) + week_group_sql = "toMonday(toTimeZone(toDateTime64(time, 3), #{quoted_timezone}))" + lag_sql = "lagInFrame(time, 1, time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time, id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" - rows = Heartbeat.connection.select_all(<<~SQL.squish) - SELECT TO_CHAR(week_group, 'YYYY-MM-DD') AS week_key, + rows = Clickhouse::Heartbeat.connection.select_all(<<~SQL.squish) + SELECT toString(week_group) AS week_key, grouped_time, - COALESCE(SUM(diff), 0)::integer AS duration + SUM(diff) AS duration FROM ( SELECT project AS grouped_time, #{week_group_sql} AS week_group, - CASE - WHEN LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time, id) IS NULL THEN 0 - ELSE LEAST( - time - LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time, id), - #{Heartbeat.heartbeat_timeout_duration.to_i} - ) - END AS diff - FROM (#{relation_sql}) dashboard_heartbeats - ) diffs + least(time - #{lag_sql}, #{Clickhouse::Heartbeat.heartbeat_timeout_duration.to_i}) AS diff + FROM (#{relation_sql}) AS dashboard_heartbeats + ) AS diffs GROUP BY week_group, grouped_time ORDER BY week_key DESC, grouped_time SQL rows.each do |row| - result[row["week_key"]][row["grouped_time"]] = row["duration"].to_i + result[row["week_key"]][row["grouped_time"]] = row["duration"].to_f.round end result @@ -108,52 +94,38 @@ def weekly_project_stats(user:, scope:) def today_stats_snapshot(user:, scope:) Time.use_zone(user.timezone) do - timeout = Heartbeat.heartbeat_timeout_duration.to_i - today_sql = scope.today.to_sql - - rows = Heartbeat.connection.select_all(<<~SQL.squish).to_a - WITH today_rows AS (#{today_sql}), - duration_calc AS ( - SELECT - CASE WHEN LAG(time) OVER (ORDER BY time, id) IS NULL THEN 0 - ELSE LEAST(time - LAG(time) OVER (ORDER BY time, id), #{timeout}) END AS diff - FROM today_rows - WHERE time IS NOT NULL AND time >= 0 AND time <= 253402300799 - ), - total_duration AS (SELECT COALESCE(SUM(diff), 0)::integer AS total FROM duration_calc) - SELECT DISTINCT - language, - editor, - COUNT(*) OVER (PARTITION BY language) AS language_count, - COUNT(*) OVER (PARTITION BY editor) AS editor_count, - (SELECT total FROM total_duration) AS total_duration - FROM today_rows - SQL - - language_categories = rows - .map { |row| [ row["language"], row["language_count"].to_i ] } + today_scope = scope.today + + language_categories = today_scope.group(:language).count .reject { |language, _| language.blank? } - .uniq - .group_by { |language, _| language.categorize_language } + .group_by { |(language, _)| language.categorize_language } .transform_values { |pairs| pairs.sum { |_, count| count } } .reject { |category, _| category.blank? } .sort_by { |_, count| -count } .map(&:first) - editor_keys = rows - .map { |row| [ row["editor"], row["editor_count"].to_i ] } + editor_keys = today_scope.group(:editor).count .reject { |editor, _| editor.blank? } - .uniq .sort_by { |_, count| -count } .map(&:first) { timezone: user.timezone, today_date: Date.current.iso8601, - todays_duration_seconds: rows.first&.fetch("total_duration").to_i, + todays_duration_seconds: today_scope.duration_seconds.to_i, todays_language_categories: language_categories, todays_editor_keys: editor_keys } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + { + timezone: user.timezone, + today_date: Date.current.iso8601, + todays_duration_seconds: 0, + todays_language_categories: [], + todays_editor_keys: [] + } end end @@ -194,8 +166,6 @@ def activity_graph_date_range(timezone) end end - # Live aggregate snapshot used by the filtered (non-rollup) dashboard path. - # Returns the same shape as the rollup-derived aggregate snapshot. def aggregate_query_snapshot(user:, scope:) { total_time: scope.duration_seconds, @@ -203,6 +173,15 @@ def aggregate_query_snapshot(user:, scope:) grouped_durations: grouped_durations_snapshot(scope), weekly_project_stats: weekly_project_stats(user: user, scope: scope) } + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") + + { + total_time: 0, + total_heartbeats: 0, + grouped_durations: GROUPED_DIMENSIONS.index_with { {} }, + weekly_project_stats: week_ranges(user.timezone).to_h { |week_key, *_| [ week_key, {} ] } + } end # Reject project entries that should not appear in dashboard summaries. diff --git a/app/services/dashboard_rollup_refresh_service.rb b/app/services/dashboard_rollup_refresh_service.rb deleted file mode 100644 index f398512c7..000000000 --- a/app/services/dashboard_rollup_refresh_service.rb +++ /dev/null @@ -1,92 +0,0 @@ -class DashboardRollupRefreshService < ApplicationService - GROUPED_DIMENSIONS = DashboardData::Snapshots::GROUPED_DIMENSIONS - WEEKLY_PROJECT_DIMENSION = DashboardData::Snapshots::WEEKLY_PROJECT_DIMENSION - - def initialize(user:) - @user = user - @scope = user.heartbeats - end - - def call - now = Time.current - records = [ - build_record(dimension: DashboardRollup::TOTAL_DIMENSION, bucket: nil, - total_seconds: @scope.duration_seconds, now:, - source_heartbeats_count: @scope.count, - source_max_heartbeat_time: @scope.maximum(:time)), - build_record(dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION, bucket: nil, - total_seconds: 0, now:, payload: filter_options_payload), - build_record(dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION, bucket: nil, - total_seconds: 0, now:, - payload: DashboardData::Snapshots.activity_graph_snapshot(user: @user, scope: @scope)), - build_record(dimension: DashboardRollup::TODAY_STATS_DIMENSION, bucket: nil, - total_seconds: 0, now:, - payload: DashboardData::Snapshots.today_stats_snapshot(user: @user, scope: @scope)) - ] - - GROUPED_DIMENSIONS.each do |dimension| - grouped_durations(dimension).each do |bucket, total_seconds| - records << build_record(dimension:, bucket:, total_seconds:, now:) - end - end - DashboardData::Snapshots.weekly_project_stats(user: @user, scope: @scope).each do |week_key, projects| - projects.each do |project, total_seconds| - records << build_record(dimension: WEEKLY_PROJECT_DIMENSION, - bucket: [ week_key, project ].to_json, total_seconds:, now:) - end - end - DashboardData::Snapshots.project_details_snapshot(scope: @scope).each do |project, details| - records << build_record( - dimension: DashboardRollup::PROJECT_DETAILS_DIMENSION, bucket: project, - total_seconds: details.fetch(:total_seconds), now:, - source_heartbeats_count: details.fetch(:total_heartbeats), - source_max_heartbeat_time: details.fetch(:last_heartbeat), - payload: { - first_heartbeat: details.fetch(:first_heartbeat), - last_heartbeat: details.fetch(:last_heartbeat), - languages: details.fetch(:languages) - } - ) - end - - DashboardRollup.transaction do - DashboardRollup.where(user_id: @user.id).delete_all - DashboardRollup.insert_all!(records) - end - DashboardRollup.clear_dirty(@user.id) - end - - private - - def build_record( - dimension:, - bucket:, - total_seconds:, - now:, - source_heartbeats_count: nil, - source_max_heartbeat_time: nil, - payload: nil - ) - { - user_id: @user.id, - dimension: dimension.to_s, - bucket_value: bucket.to_s, - bucket_value_present: !bucket.nil?, - total_seconds: total_seconds.to_i, - source_heartbeats_count:, - source_max_heartbeat_time:, - payload:, - created_at: now, - updated_at: now - } - end - - def grouped_durations(dimension) - return DashboardData::Snapshots.project_grouped_durations(@scope) if dimension == :project - Heartbeat.attributed_durations_by(@scope, dimension) - end - - def filter_options_payload - GROUPED_DIMENSIONS.index_with { |d| @scope.distinct.pluck(d).compact_blank.sort } - end -end diff --git a/app/services/dashboard_stats.rb b/app/services/dashboard_stats.rb index b3be4d12c..c18dc8b4b 100644 --- a/app/services/dashboard_stats.rb +++ b/app/services/dashboard_stats.rb @@ -14,33 +14,20 @@ def initialize(user:, params: ActionController::Parameters.new) def filterable_dashboard_data interval = params[:interval] - return build_filterable_dashboard_data(interval) if rollup_eligible? - key = [ user ] + FILTERS.map { |field| params[field] } + [ interval.to_s, params[:from], params[:to] ] Rails.cache.fetch(key, expires_in: 5.minutes) { build_filterable_dashboard_data(interval) } end - def activity_graph_data - row = rollup_fragment_row(DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - return activity_graph_from_rollup(row) if activity_graph_rollup_valid?(row) - schedule_rollup_refresh(wait: 0.seconds) if rollups_available? - live_activity_graph_data - end - - def today_stats_data - row = rollup_fragment_row(DashboardRollup::TODAY_STATS_DIMENSION) - return today_stats_from_rollup(row) if today_stats_rollup_valid?(row) - schedule_rollup_refresh(wait: 0.seconds) if rollups_available? - live_today_stats_data - end + def activity_graph_data = live_activity_graph_data + def today_stats_data = live_today_stats_data # ---- Building blocks ---------------------------------------------------- # Public so tests (and ProfileStatsService) can inspect/override. def build_filterable_dashboard_data(interval) archived = user.project_repo_mappings.archived.pluck(:project_name) - raw_filter_options = raw_filter_options(archived: archived) - result = rollup_result(raw_filter_options, archived) || query_result(raw_filter_options, archived) + raw_filter_options = live_raw_filter_options + result = query_result(raw_filter_options, archived) result[:selected_interval] = interval.to_s result[:selected_from] = params[:from].to_s result[:selected_to] = params[:to].to_s @@ -48,36 +35,25 @@ def build_filterable_dashboard_data(interval) result end - def raw_filter_options(archived: []) - (rollup_eligible? && rollup_filter_options) || live_raw_filter_options - end - def live_raw_filter_options cache_keys = FILTERS.index_with { |field| "user_#{user.id}_dashboard_filter_options_#{field}_#{FILTER_OPTIONS_CACHE_VERSION}" } reverse_lookup = cache_keys.invert cached = Rails.cache.fetch_multi(*cache_keys.values, expires_in: 15.minutes) do |cache_key| - user.heartbeats.distinct.pluck(reverse_lookup.fetch(cache_key)).compact_blank - end - - cache_keys.transform_values { |cache_key| cached.fetch(cache_key, []) } - end - - def rollup_filter_options - return unless rollups_available? + field = reverse_lookup.fetch(cache_key) + rows = heartbeats_scope.where.not(field => nil).distinct.select(field).to_sql + Clickhouse::Heartbeat.connection.select_all(rows).map { |row| row[field.to_s] }.compact_blank.sort + rescue ActiveRecord::ActiveRecordError => e + raise unless e.message.include?("undefined method 'map' for nil") - row = rollup_fragment_row(DashboardRollup::FILTER_OPTIONS_DIMENSION) - payload = row&.payload - unless payload.is_a?(Hash) && FILTERS.all? { |field| payload[field.to_s].is_a?(Array) || payload[field].is_a?(Array) } - schedule_rollup_refresh(wait: 0.seconds) - return + [] end - FILTERS.index_with { |field| Array(payload[field.to_s] || payload[field]) } + cache_keys.transform_values { |cache_key| cached.fetch(cache_key, []) } end def query_result(raw_filter_options, archived) - hb = user.heartbeats + hb = heartbeats_scope result = filter_options_result(raw_filter_options, archived) h = ApplicationController.helpers @@ -103,17 +79,6 @@ def query_result(raw_filter_options, archived) result end - def rollup_result(raw_filter_options, archived) - snapshot = aggregate_rollup_snapshot - return unless snapshot - - result = filter_options_result(raw_filter_options, archived) - Time.use_zone(user.timezone) do - DashboardData::Snapshots.fill_aggregate_result(result: result, snapshot: snapshot, archived: archived, helpers: ApplicationController.helpers) - end - result - end - def filter_options_result(raw_filter_options, archived) h = ApplicationController.helpers FILTERS.each_with_object({}) do |field, result| @@ -130,48 +95,7 @@ def filter_options_result(raw_filter_options, archived) end end - def aggregate_rollup_snapshot - return unless rollups_available? && rollup_eligible? - - total_row = rollup_total_row - unless total_row - schedule_rollup_refresh(wait: 0.seconds) - return - end - schedule_rollup_refresh(wait: 0.seconds) if aggregate_rollup_stale?(total_row) - - { - total_time: total_row.total_seconds, - total_heartbeats: total_row.source_heartbeats_count.to_i, - grouped_durations: FILTERS.index_with { |field| - rollup_rows_by_dimension.fetch(field.to_s, []).to_h { |row| [ row.bucket, row.total_seconds ] } - }, - weekly_project_stats: rollup_weekly_project_stats(rollup_rows_by_dimension.fetch(WEEKLY_PROJECT_DIMENSION, [])) - } - end - - def rollup_eligible? - params[:interval].blank? && params[:from].blank? && params[:to].blank? && - FILTERS.none? { |field| params[field].present? } - end - - def rollups_available? - DashboardRollup.table_exists? - rescue ActiveRecord::StatementInvalid - false - end - - def rollup_rows - return [] unless rollups_available? - @rollup_rows ||= DashboardRollup.where(user_id: user.id).to_a - end - - def rollup_rows_by_dimension = @rollup_rows_by_dimension ||= rollup_rows.group_by(&:dimension) - def rollup_fragment_row(dimension) = rollup_rows_by_dimension.fetch(dimension.to_s, []).first - def rollup_total_row = @rollup_total_row ||= rollup_rows.find(&:total_dimension?) - def rollup_source_max_heartbeat_time = rollup_time_fingerprint(user.heartbeats.maximum(:time)) - def rollup_time_fingerprint(timestamp) = timestamp.nil? ? nil : (timestamp * 1_000_000).round - def today_date = Time.use_zone(user.timezone) { Date.current.iso8601 } + def heartbeats_scope = Clickhouse::Heartbeat.for_user(user) def activity_graph_date_range(timezone) = DashboardData::Snapshots.activity_graph_date_range(timezone) def grouped_durations_snapshot(scope) = DashboardData::Snapshots.grouped_durations_snapshot(scope) def project_grouped_durations(scope) = DashboardData::Snapshots.project_grouped_durations(scope) @@ -179,60 +103,14 @@ def weekly_project_stats(scope, _timezone = user.timezone) = DashboardData::Snap def week_ranges = DashboardData::Snapshots.week_ranges(user.timezone) def today_stats_snapshot(scope) = DashboardData::Snapshots.today_stats_snapshot(user: user, scope: scope) - def aggregate_rollup_stale?(total_row) - DashboardRollup.dirty?(user.id) || - rollup_time_fingerprint(total_row.source_max_heartbeat_time) != rollup_source_max_heartbeat_time - end - - def schedule_rollup_refresh(wait:) - return if @rollup_refresh_scheduled - DashboardRollupRefreshJob.schedule_for(user.id, wait: wait) - @rollup_refresh_scheduled = true - end - - def activity_graph_rollup_valid?(row) - payload = row&.payload - return false unless payload.is_a?(Hash) - start_date, end_date = activity_graph_date_range(user.timezone) - payload["timezone"] == user.timezone && payload["start_date"] == start_date && - payload["end_date"] == end_date && payload["duration_by_date"].is_a?(Hash) - end - - def today_stats_rollup_valid?(row) - payload = row&.payload - return false unless payload.is_a?(Hash) - payload["timezone"] == user.timezone && payload["today_date"] == today_date && - payload.key?("todays_duration_seconds") && - payload["todays_language_categories"].is_a?(Array) && - payload["todays_editor_keys"].is_a?(Array) - end - def live_activity_graph_data timezone = user.timezone start_date, end_date = activity_graph_date_range(timezone) durations = Rails.cache.fetch(user.activity_graph_cache_key(timezone), expires_in: 1.minute) do - Time.use_zone(timezone) { user.heartbeats.daily_durations(user_timezone: timezone).to_h } + Time.use_zone(timezone) { heartbeats_scope.daily_durations(user_timezone: timezone).to_h } end DashboardData::Snapshots.activity_graph_result(start_date: start_date, end_date: end_date, duration_by_date: durations, timezone: timezone) end - def activity_graph_from_rollup(row) - payload = row.payload || {} - DashboardData::Snapshots.activity_graph_result( - start_date: payload["start_date"], end_date: payload["end_date"], - duration_by_date: payload["duration_by_date"], timezone: payload["timezone"] || user.timezone - ) - end - - def live_today_stats_data = DashboardData::Snapshots.today_stats_display(today_stats_snapshot(user.heartbeats), helpers: ApplicationController.helpers) - def today_stats_from_rollup(row) = DashboardData::Snapshots.today_stats_display(row.payload, helpers: ApplicationController.helpers) - - def rollup_weekly_project_stats(rows) - result = week_ranges.to_h { |week_key, *_| [ week_key, {} ] } - rows.each do |row| - week_key, project = JSON.parse(row.bucket_value) - result[week_key][project] = row.total_seconds if result.key?(week_key) - end - result - end + def live_today_stats_data = DashboardData::Snapshots.today_stats_display(today_stats_snapshot(heartbeats_scope), helpers: ApplicationController.helpers) end diff --git a/app/services/heartbeat_import_service.rb b/app/services/heartbeat_import_service.rb index de832d503..7ec4f058e 100644 --- a/app/services/heartbeat_import_service.rb +++ b/app/services/heartbeat_import_service.rb @@ -10,8 +10,8 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva flush = lambda do next if heartbeat_batch.empty? - result = HeartbeatIngest.call(user:, mode: :import, heartbeats: heartbeat_batch.values, - user_agents_by_id:, schedule_rollup_refresh: false) + result = HeartbeatIngest.call(user:, mode: :import, heartbeats: heartbeat_batch.values.map { |entry| entry[:heartbeat] }, + user_agents_by_id:) imported_count += result.persisted_count errors.concat(result.errors) heartbeat_batch.clear @@ -23,7 +23,10 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva begin attrs = HeartbeatIngest.normalize_imported_heartbeat(user:, heartbeat: hb, user_agents_by_id:) - heartbeat_batch[attrs[:fields_hash]] = hb + existing = heartbeat_batch[attrs[:fields_hash]] + if existing.nil? || attrs[:time].to_f >= existing[:time] + heartbeat_batch[attrs[:fields_hash]] = { heartbeat: hb, time: attrs[:time].to_f } + end flush.call if heartbeat_batch.size >= BATCH_SIZE rescue => e errors << { heartbeat: hb, error: e.message } @@ -35,7 +38,6 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva raise StandardError, "Expected a heartbeat export JSON file." if total_count.zero? flush.call - HeartbeatIngest.schedule_rollup_refresh(user:) if imported_count.positive? elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time { success: true, imported_count:, total_count:, diff --git a/app/services/heartbeat_ingest.rb b/app/services/heartbeat_ingest.rb index 49c68bc3c..09f3ab73f 100644 --- a/app/services/heartbeat_ingest.rb +++ b/app/services/heartbeat_ingest.rb @@ -6,26 +6,26 @@ class HeartbeatIngest EPOCH_SANE_MIN = 1_000_000_000 EPOCH_SANE_MAX = 2_000_000_000 - # The heartbeats dedup unique index changes from (fields_hash) to - # (fields_hash, time_epoch) during the hypertable migration. - UNIQUE_BY_LEGACY = [ :fields_hash ].freeze - UNIQUE_BY_COMPOSITE = %i[fields_hash time_epoch].freeze + ATTRIBUTE_KEYS = %i[ + user_id time project branch entity category editor language machine + operating_system type user_agent ip_address dependencies lineno lines + cursorpos line_additions line_deletions project_root_count is_write + source_type ja4_id + ].freeze Result = Data.define(:total_count, :persisted_count, :duplicate_count, :failed_count, :errors, :items) Item = Data.define(:heartbeat, :status, :error) def self.call(...) = new(...).call - def self.schedule_rollup_refresh(user:) = DashboardRollupRefreshJob.schedule_for(user.id) def self.normalize_imported_heartbeat(user:, heartbeat:, user_agents_by_id: {}) = new(user:, mode: :import, heartbeats: [], user_agents_by_id:).send(:normalize_imported_heartbeat, heartbeat) - def initialize(user:, mode:, heartbeats:, request_context: {}, user_agents_by_id: {}, schedule_rollup_refresh: true) + def initialize(user:, mode:, heartbeats:, request_context: {}, user_agents_by_id: {}) @user = user @mode = mode @heartbeats = heartbeats @request_context = request_context.with_indifferent_access @user_agents_by_id = user_agents_by_id - @schedule_rollup_refresh = schedule_rollup_refresh end def call @@ -39,25 +39,38 @@ def call private def ingest_direct - items, errors = [], [] - persisted_count = duplicate_count = 0 + items, errors, rows = [], [], [] + seen_hashes = {} + duplicate_count = 0 last_language = nil @heartbeats.each do |heartbeat| attrs = normalize_direct_heartbeat(heartbeat, last_language:) - persisted, duplicate = persist_direct_heartbeat(attrs) + fields_hash = Clickhouse::Heartbeat.generate_fields_hash(attrs) last_language = attrs[:language] if attrs[:language].present? - duplicate ? duplicate_count += 1 : persisted_count += 1 - items << Item.new(heartbeat: persisted, status: :accepted, error: nil) + + # In-batch dedup only; cross-request duplicates share the same + # (user_id, time, fields_hash) ORDER BY key and collapse in ClickHouse. + row = seen_hashes[fields_hash] + if row + duplicate_count += 1 + else + row = build_row(attrs, fields_hash:) + seen_hashes[fields_hash] = row + rows << row + end + items << Item.new(heartbeat: row, status: :accepted, error: nil) queue_project_mapping(attrs[:project]) rescue => e errors << { heartbeat: heartbeat, error: e.message, type: e.class.name } items << Item.new(heartbeat: nil, status: :failed, error: e) end + Clickhouse::HeartbeatWriter.insert_rows(rows) if rows.any? + Result.new( total_count: @heartbeats.length, - persisted_count:, + persisted_count: rows.length, duplicate_count:, failed_count: errors.length, errors:, @@ -71,8 +84,8 @@ def normalize_direct_heartbeat(heartbeat, last_language:) source_type = attrs[:entity] == "test.txt" ? :test_entry : :direct_entry if attrs[:language] == LAST_LANGUAGE_SENTINEL - attrs[:language] = last_language || @user.heartbeats - .where.not(language: [ nil, "", LAST_LANGUAGE_SENTINEL ]).order(time: :desc).pick(:language) + attrs[:language] = last_language || Clickhouse::Heartbeat.for_user(@user) + .where.not(language: [ nil, "", LAST_LANGUAGE_SENTINEL ]).order(time: :desc).limit(1).pick(:language) end if attrs[:language].blank? || attrs[:language] == "Unknown" @@ -93,25 +106,12 @@ def normalize_direct_heartbeat(heartbeat, last_language:) editor: parsed_ua[:editor], operating_system: parsed_ua[:os], machine: @request_context[:machine] - ).slice(*Heartbeat.column_names.map(&:to_sym)) + ).slice(*ATTRIBUTE_KEYS) end - def persist_direct_heartbeat(attrs) - fields_hash = Heartbeat.generate_fields_hash(Heartbeat.new(attrs).attributes) - existing = @user.heartbeats.find_by(fields_hash: fields_hash) - return [ existing, true ] if existing - + def build_row(attrs, fields_hash:) now = Time.current - result = with_heartbeat_unique_by do |unique_by| - Heartbeat.insert( - attrs.merge(fields_hash:, created_at: now, updated_at: now, **partition_attrs(attrs[:time])), - unique_by:, returning: Heartbeat.column_names - ) - end - - persisted = result.any? ? Heartbeat.new(result.first) : @user.heartbeats.find_by!(fields_hash:) - self.class.schedule_rollup_refresh(user: @user) if result.any? && @schedule_rollup_refresh - [ persisted, !result.any? ] + Clickhouse::HeartbeatWriter.send(:shape_row, attrs.merge(fields_hash:, created_at: now, updated_at: now)) end def ingest_import @@ -129,7 +129,6 @@ def ingest_import end persisted_count = flush_import_batch(seen_hashes) - self.class.schedule_rollup_refresh(user: @user) if persisted_count.positive? && @schedule_rollup_refresh Result.new( total_count:, @@ -168,53 +167,18 @@ def normalize_imported_heartbeat(heartbeat) cursorpos: hb[:cursorpos], dependencies: hb[:dependencies] || [], project_root_count: hb[:project_root_count], - source_type: Heartbeat.source_types.fetch("wakapi_import") + source_type: :wakapi_import } - attrs[:fields_hash] = Heartbeat.generate_fields_hash(attrs) + attrs[:fields_hash] = Clickhouse::Heartbeat.generate_fields_hash(attrs) attrs end def flush_import_batch(seen_hashes) return 0 if seen_hashes.empty? timestamp = Time.current - ActiveRecord::Base.logger.silence do - # Build records inside the retry block so a cutover-time schema refresh - # recomputes both the conflict target and the time_epoch partition column. - with_heartbeat_unique_by do |unique_by| - records = seen_hashes.values.map { |r| r.merge(created_at: timestamp, updated_at: timestamp, **partition_attrs(r[:time])) } - Heartbeat.insert_all(records, unique_by:).length - end - end - end - - def heartbeat_unique_by - time_epoch_column? ? UNIQUE_BY_COMPOSITE : UNIQUE_BY_LEGACY - end - - def time_epoch_column? = Heartbeat.column_names.include?("time_epoch") - - # The hypertable partition column must arrive populated (TimescaleDB routes to - # a chunk before row triggers fire). Bulk insert/insert_all bypass model - # callbacks, so ingest supplies time_epoch explicitly once the column exists. - # No-op on the pre-cutover / dev-and-test plain table. - def partition_attrs(time) - return {} unless time_epoch_column? && time.present? - { time_epoch: time.to_f.floor } - end - - # Rails resolves `unique_by:` against its schema cache, which goes stale the - # moment the hypertable cutover swaps the table under us. Refresh the cache - # and retry once so in-flight processes self-heal without a restart. - def with_heartbeat_unique_by - yield heartbeat_unique_by - rescue ActiveRecord::StatementInvalid, ArgumentError => e - Heartbeat.reset_column_information - # Inside an open (now aborted) transaction a retry cannot succeed; re-raise - # and let the caller retry with the already-refreshed schema cache. - raise if Heartbeat.connection.transaction_open? - - Rails.logger.warn("HeartbeatIngest unique_by fallback: #{e.class}: #{e.message}") - yield heartbeat_unique_by + rows = seen_hashes.values.map { |r| r.merge(created_at: timestamp, updated_at: timestamp) } + Clickhouse::HeartbeatWriter.insert_rows(rows) + rows.length end def normalize_epoch_time(value) diff --git a/app/services/programming_goals_progress_service.rb b/app/services/programming_goals_progress_service.rb index 6fc8ab5d8..aed3f4e6d 100644 --- a/app/services/programming_goals_progress_service.rb +++ b/app/services/programming_goals_progress_service.rb @@ -37,7 +37,7 @@ def build_progress(goal, now:) def tracked_seconds_for_goal(goal, now:) time_window = time_window_for(goal.period, now: now) - scope = user.heartbeats.where(time: time_window.begin.to_i..time_window.end.to_i) + scope = Clickhouse::Heartbeat.for_user(user).where(time: time_window.begin.to_i..time_window.end.to_i) scope = scope.where(project: goal.projects) if goal.projects.any? if goal.languages.any? @@ -52,7 +52,7 @@ def tracked_seconds_for_goal(goal, now:) scope = scope.where(language: matching_languages) end - scope.duration_seconds.to_i + Clickhouse::Heartbeat.duration_seconds(scope).to_i end def time_window_for(period, now:) diff --git a/app/services/project_stats_query.rb b/app/services/project_stats_query.rb index fb522cfee..7164239ef 100644 --- a/app/services/project_stats_query.rb +++ b/app/services/project_stats_query.rb @@ -26,10 +26,6 @@ def project_details(names: nil) requested_names = Array(names.presence || parse_csv(@params[:projects])) .map { |n| n.to_s.strip }.reject(&:blank?).uniq - if requested_names.empty? && rollup_eligible? && (rollup_details = rollup_project_details) - return rollup_details - end - query = scoped_heartbeats(stats_start_time, stats_end_time) query = query.where(project: requested_names) if requested_names.any? @@ -77,47 +73,15 @@ def build_project_row( def scoped_heartbeats(start_time, end_time) start_ts = timestamp_value(start_time) end_ts = timestamp_value(end_time) - return @user.heartbeats.none if start_ts.nil? || end_ts.nil? + return Clickhouse::Heartbeat.none if start_ts.nil? || end_ts.nil? - @user.heartbeats.with_valid_timestamps.where.not(project: [ nil, "" ]).where(time: start_ts..end_ts) + Clickhouse::Heartbeat.for_user(@user).with_valid_timestamps.where.not(project: [ nil, "" ]).where(time: start_ts..end_ts) end def archived_project_names @archived_project_names ||= @user.project_repo_mappings.archived.pluck(:project_name) end - def rollup_project_details - rows = DashboardRollup.where(user_id: @user.id, dimension: DashboardRollup::PROJECT_DETAILS_DIMENSION, bucket_value_present: true).to_a - return if rows.empty? - - DashboardRollupRefreshJob.schedule_for(@user.id, wait: 0.seconds) if DashboardRollup.dirty?(@user.id) - - details_by_project = rows.index_by(&:bucket) - details_by_project.delete("") - return if details_by_project.empty? - - repo_mappings = @user.project_repo_mappings.where(project_name: details_by_project.keys).index_by(&:project_name) - - details_by_project.filter_map { |name, rollup| - next if !@include_archived && repo_mappings[name]&.archived? - payload = rollup.payload.to_h - build_project_row( - name: name, stat: {}, repo_mapping: repo_mappings[name], - total_seconds: rollup.total_seconds.to_i, - total_heartbeats: rollup.source_heartbeats_count.to_i, - languages: Array(payload["languages"]).compact_blank, - first_heartbeat: payload["first_heartbeat"], - last_heartbeat: payload["last_heartbeat"] - ) - }.sort_by { |project| -project[:total_seconds] } - end - - def rollup_eligible? - @params[:projects].blank? && - %i[start start_date end end_date].none? { |key| @params[key].present? } && - timestamp_value(@default_stats_start).to_f.zero? - end - def discovery_start_time = parse_time([ :since, :start, :start_date ], default: @default_discovery_start) def discovery_end_time = parse_time([ :until, :until_date, :end, :end_date ], default: @default_discovery_end) def stats_start_time = parse_time([ :start, :start_date ], default: @default_stats_start) diff --git a/app/services/project_stats_service.rb b/app/services/project_stats_service.rb index 881560bb4..c82ec8bda 100644 --- a/app/services/project_stats_service.rb +++ b/app/services/project_stats_service.rb @@ -17,13 +17,14 @@ def call(only: FIELDS) attr_reader :hb def h = ApplicationController.helpers + def heartbeat_model = hb.klass - def total_time = @total_time ||= hb.duration_seconds + def total_time = @total_time ||= heartbeat_model.duration_seconds(hb) def file_count = hb.select(:entity).distinct.count def grouped(field, n, normalize: ->(k) { k.to_s }, display: nil) - result = Heartbeat.attributed_durations_by(hb, field).each_with_object({}) do |(raw, dur), agg| + result = heartbeat_model.attributed_durations_by(hb, field).each_with_object({}) do |(raw, dur), agg| k = normalize.call(raw) agg[k] = (agg[k] || 0) + dur end.sort_by { |_, d| -d }.first(n) @@ -47,11 +48,11 @@ def os_stats def category_stats = grouped(:category, 10) def file_stats - Heartbeat.attributed_durations_by(hb, :entity) + heartbeat_model.attributed_durations_by(hb, :entity) .reject { |_, dur| dur < 60 } .sort_by { |_, d| -d }.first(50) .map { |entity, dur| [ h.shorten_file_path(entity), dur ] } end - def branch_stats = Heartbeat.attributed_durations_by(hb, :branch).sort_by { |_, d| -d }.first(10) + def branch_stats = heartbeat_model.attributed_durations_by(hb, :branch).sort_by { |_, d| -d }.first(10) end diff --git a/app/services/timeline_service.rb b/app/services/timeline_service.rb index 203a8d7a5..b8ce0aa37 100644 --- a/app/services/timeline_service.rb +++ b/app/services/timeline_service.rb @@ -18,9 +18,8 @@ def timeline_data day_start = date.in_time_zone(user_tz).beginning_of_day.to_f day_end = date.in_time_zone(user_tz).end_of_day.to_f - total_coded_time_seconds = Heartbeat.where(user_id: user.id, deleted_at: nil) - .where("time >= ? AND time <= ?", day_start, day_end) - .duration_seconds + total_scope = Clickhouse::Heartbeat.for_user(user).where("time >= ? AND time <= ?", day_start, day_end) + total_coded_time_seconds = Clickhouse::Heartbeat.duration_seconds(total_scope) hbs = (heartbeats_by_user_id[user.id] || []).select { |hb| hb.time >= day_start && hb.time <= day_end } @@ -51,11 +50,11 @@ def mappings_by_user_project end def heartbeats_by_user_id - @heartbeats_by_user_id ||= Heartbeat - .where(user_id: users_by_id.keys, deleted_at: nil) + @heartbeats_by_user_id ||= Clickhouse::Heartbeat + .where(user_id: users_by_id.keys) .where("time >= ? AND time <= ?", date.beginning_of_day.to_f - 24.hours.to_i, date.end_of_day.to_f + 24.hours.to_i) .select(:id, :user_id, :time, :entity, :project, :editor, :language) - .order(:user_id, :time).to_a.group_by(&:user_id) + .order(:user_id, :time, :id).to_a.group_by(&:user_id) end def calculate_spans(user, heartbeats) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index a03f4045c..e6c34658b 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -215,8 +215,8 @@

Build <%= link_to Rails.application.config.git_version, Rails.application.config.commit_link, class: 'text-inherit underline opacity-80 hover:opacity-100 transition-opacity duration-200' %> from <%= time_ago_in_words(Rails.application.config.server_start_time) %> ago. - <%= pluralize(Heartbeat.recent_count, 'heartbeat') %> - (<%= Heartbeat.recent_imported_count %> imported) in the past 24 hours.<% if current_user&.can_view_query_stats? %> (DB: <%= pluralize(QueryCount::Counter.counter, 'query') %>, <%= QueryCount::Counter.counter_cache %> cached)<% end %> + <%= pluralize(Clickhouse::Heartbeat.recent_count, 'heartbeat') %> + (<%= Clickhouse::Heartbeat.recent_imported_count %> imported) in the past 24 hours.<% if current_user&.can_view_query_stats? %> (DB: <%= pluralize(QueryCount::Counter.counter, 'query') %>, <%= QueryCount::Counter.counter_cache %> cached)<% end %>

<% if session[:impersonater_user_id] %> <%= link_to 'Stop impersonating', stop_impersonating_path, class: 'text-primary font-bold hover:text-red transition-colors duration-200', data: { turbo_prefetch: 'false' } %> diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 39cb1a651..4d0a34d66 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -3,48 +3,71 @@ { "warning_type": "SQL Injection", "warning_code": 0, - "fingerprint": "06ca3650eaeb8d28e062c1c6dcbeab95fb2ccd0c5bb49165ad469bcb6b791d3e", + "fingerprint": "62b399e453470663b181f03607cfe7a32dfab3ebf714b907dbe4ccbb540188df", "check_name": "SQL", "message": "Possible SQL injection", - "file": "app/models/concerns/heartbeatable.rb", - "line": 175, + "file": "app/models/clickhouse/heartbeat_writer.rb", + "line": 53, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "Arel.sql(\"DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE '#{(user_timezone or \"UTC\")}')\")", + "code": "Clickhouse::Heartbeat.connection.execute(...)" , "render_path": null, "location": { "type": "method", - "class": "Heartbeatable", - "method": "daily_durations" + "class": "Clickhouse::HeartbeatWriter", + "method": "soft_delete_user_heartbeats!" }, - "user_input": "user_timezone", + "user_input": "column_list(Clickhouse::Heartbeat.connection)", "confidence": "Medium", "cwe_id": [ 89 ], - "note": "" + "note": "Identifiers come from WRITABLE_COLUMNS and are quoted; user IDs are cast with to_i." }, { "warning_type": "SQL Injection", "warning_code": 0, - "fingerprint": "db41fbf90d0feb7b7c1c11545d498447162cf5e54e091d2ddcfcaba28b2513f6", + "fingerprint": "3e072caffbc9db064555f78b2b32edaa9f9b2c62843181cd03e13600bc310038", "check_name": "SQL", "message": "Possible SQL injection", - "file": "app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb", - "line": 32, + "file": "app/models/clickhouse/heartbeat_writer.rb", + "line": 72, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "Heartbeat.where(:id => chunk.map do\n index = 1\nputs(\"Processing heartbeat #{heartbeat.id} (#{1} of #{batch.size})\")\nfield_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)\nputs(\"Field hash: #{Heartbeat.generate_fields_hash(heartbeat.attributes)}\")\n[heartbeat.id, Heartbeat.generate_fields_hash(heartbeat.attributes)]\n end.map(&:first)).update_all(\"fields_hash = CASE #{chunk.map do\n index = 1\nputs(\"Processing heartbeat #{heartbeat.id} (#{1} of #{batch.size})\")\nfield_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)\nputs(\"Field hash: #{Heartbeat.generate_fields_hash(heartbeat.attributes)}\")\n[heartbeat.id, Heartbeat.generate_fields_hash(heartbeat.attributes)]\n end.map do\n \"WHEN id = #{id} THEN '#{hash}'\"\n end.join(\" \")} END\")", + "code": "Clickhouse::Heartbeat.connection.execute(...)" , "render_path": null, "location": { "type": "method", - "class": "OneTime::GenerateUniqueHeartbeatHashesJob", - "method": "perform" + "class": "Clickhouse::HeartbeatWriter", + "method": "merge_user_heartbeats!" }, - "user_input": "Heartbeat.generate_fields_hash(heartbeat.attributes)", - "confidence": "High", + "user_input": "column_list(Clickhouse::Heartbeat.connection)", + "confidence": "Medium", + "cwe_id": [ + 89 + ], + "note": "Identifiers come from WRITABLE_COLUMNS and are quoted; user IDs are cast with to_i." + }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "256e58728e5f1b2442828e109d75dbcabd9cf85b2b839d1294d88690ebd6cd26", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/clickhouse/heartbeat_writer.rb", + "line": 85, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "Clickhouse::Heartbeat.connection.execute(...)" , + "render_path": null, + "location": { + "type": "method", + "class": "Clickhouse::HeartbeatWriter", + "method": "merge_user_heartbeats!" + }, + "user_input": "column_list(Clickhouse::Heartbeat.connection)", + "confidence": "Medium", "cwe_id": [ 89 ], - "note": "" + "note": "Identifiers come from WRITABLE_COLUMNS and are quoted; user IDs are cast with to_i." } ], "brakeman_version": "7.0.2" diff --git a/config/database.yml b/config/database.yml index ae7735683..5a6f604c4 100644 --- a/config/database.yml +++ b/config/database.yml @@ -14,6 +14,18 @@ development: encoding: unicode url: <%= ENV['SAILORS_LOG_DATABASE_URL'] %> replica: true + clickhouse: + adapter: clickhouse + host: <%= ENV.fetch("CLICKHOUSE_HOST", "clickhouse") %> + port: <%= ENV.fetch("CLICKHOUSE_PORT", 8123) %> + database: <%= ENV.fetch("CLICKHOUSE_DATABASE", "hackatime_development") %> + username: <%= ENV.fetch("CLICKHOUSE_USER", "default") %> + password: <%= ENV.fetch("CLICKHOUSE_PASSWORD", "") %> + migrations_paths: db/clickhouse + # SQL structure dump (SHOW CREATE TABLE) instead of the Ruby schema dumper, + # which lossily downgrades Float64->Float32 and FixedString->String. Postgres + # keeps the default :ruby format. + schema_format: sql # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -29,6 +41,16 @@ test: database: app_test url: <%= ENV['TEST_DATABASE_URL'] %> replica: true + clickhouse: + adapter: clickhouse + host: <%= ENV.fetch("CLICKHOUSE_HOST", "clickhouse") %> + port: <%= ENV.fetch("CLICKHOUSE_PORT", 8123) %> + database: <%= ENV.fetch("CLICKHOUSE_TEST_DATABASE", "hackatime_test") %> + username: <%= ENV.fetch("CLICKHOUSE_USER", "default") %> + password: <%= ENV.fetch("CLICKHOUSE_PASSWORD", "") %> + migrations_paths: db/clickhouse + # See development note: SQL structure dump avoids the lossy Ruby dumper. + schema_format: sql # Store production database in the storage/ directory, which by default # is mounted as a persistent Docker volume in config/deploy.yml. @@ -54,3 +76,11 @@ production: adapter: sqlite3 database: storage/production_cable.sqlite3 migrations_paths: db/cable_migrate + clickhouse: + adapter: clickhouse + url: <%= ENV['CLICKHOUSE_URL'] %> + migrations_paths: db/clickhouse + schema_format: sql + # Skip schema management (db:prepare etc.) until the production ClickHouse + # deployment exists, so boot/deploy don't fail on a missing CLICKHOUSE_URL. + database_tasks: <%= ENV["CLICKHOUSE_URL"].present? %> diff --git a/db/clickhouse/20260706000001_create_heartbeats.rb b/db/clickhouse/20260706000001_create_heartbeats.rb new file mode 100644 index 000000000..4e0a16820 --- /dev/null +++ b/db/clickhouse/20260706000001_create_heartbeats.rb @@ -0,0 +1,39 @@ +class CreateHeartbeats < ActiveRecord::Migration[8.1] + def change + create_table :heartbeats, + id: false, + options: "ReplacingMergeTree(version) " \ + "PARTITION BY toYYYYMM(toDateTime(time)) " \ + "ORDER BY (user_id, time, fields_hash) " \ + "SETTINGS index_granularity = 8192" do |t| + t.column :id, "UInt64", null: false, codec: "Delta(8), LZ4" + t.column :user_id, "UInt32", null: false, codec: "T64, ZSTD(1)" + t.column :time, "Float64", null: false, codec: "Gorilla, ZSTD(1)" + t.column :project, "String", codec: "ZSTD(3)" + t.column :branch, "String", codec: "ZSTD(3)" + t.column :category, "String", low_cardinality: true, codec: "ZSTD(1)" + t.column :editor, "String", low_cardinality: true, codec: "ZSTD(1)" + t.column :entity, "String", codec: "ZSTD(3)" + t.column :language, "String", low_cardinality: true, codec: "ZSTD(1)" + t.column :machine, "String", low_cardinality: true, codec: "ZSTD(1)" + t.column :operating_system, "String", low_cardinality: true, codec: "ZSTD(1)" + t.column :user_agent, "String", codec: "ZSTD(3)" + t.column :lineno, "Int32", codec: "T64, ZSTD(1)" + t.column :lines, "Int32", codec: "T64, ZSTD(1)" + t.column :cursorpos, "Int32", codec: "T64, ZSTD(1)" + t.column :line_additions, "Int32", codec: "T64, ZSTD(1)" + t.column :line_deletions, "Int32", codec: "T64, ZSTD(1)" + t.column :project_root_count, "Int32", codec: "T64, ZSTD(1)" + t.column :is_write, "Bool" + t.column :source_type, "UInt8", null: false, codec: "T64, ZSTD(1)" + t.column :ip_address, "String", codec: "ZSTD(1)" + t.column :dependencies, "String", array: true, null: false, codec: "ZSTD(3)" + t.column :ja4_id, "Int32", codec: "T64, ZSTD(1)" + t.column :fields_hash, "FixedString(32)", null: false, codec: "ZSTD(1)" + t.column :deleted_at, "DateTime64(6, 'UTC')", codec: "Delta(8), ZSTD(1)" + t.column :created_at, "DateTime64(6, 'UTC')", null: false, codec: "Delta(8), ZSTD(1)" + t.column :updated_at, "DateTime64(6, 'UTC')", null: false, codec: "Delta(8), ZSTD(1)" + t.column :version, "UInt64", null: false, codec: "Delta(8), ZSTD(1)" + end + end +end diff --git a/db/clickhouse/20260708000001_add_type_to_heartbeats.rb b/db/clickhouse/20260708000001_add_type_to_heartbeats.rb new file mode 100644 index 000000000..47e4c78a2 --- /dev/null +++ b/db/clickhouse/20260708000001_add_type_to_heartbeats.rb @@ -0,0 +1,5 @@ +class AddTypeToHeartbeats < ActiveRecord::Migration[8.1] + def change + add_column :heartbeats, :type, "String", low_cardinality: true, codec: "ZSTD(1)" + end +end diff --git a/db/clickhouse/20260709000001_rebuild_heartbeats_for_native_writes.rb b/db/clickhouse/20260709000001_rebuild_heartbeats_for_native_writes.rb new file mode 100644 index 000000000..c97fc398b --- /dev/null +++ b/db/clickhouse/20260709000001_rebuild_heartbeats_for_native_writes.rb @@ -0,0 +1,59 @@ +# Heartbeats are now written directly to ClickHouse (no Postgres copy, no +# mirror). Rebuild the table to the canonical schema shared with production: +# - fields_hash is String (matches prod; FixedString(32) padded on dev only) +# - _peerdb_* compat columns remain so the production PeerDB bridge can keep +# writing during the cutover window; `version DEFAULT _peerdb_version` makes +# CDC rows order correctly while app rows supply version explicitly. +# Dev/test data is disposable — reseed after migrating. +class RebuildHeartbeatsForNativeWrites < ActiveRecord::Migration[8.1] + def up + execute "DROP TABLE IF EXISTS heartbeats" + execute <<~SQL + CREATE TABLE heartbeats + ( + id UInt64 CODEC(Delta(8), LZ4), + user_id UInt32 CODEC(T64, ZSTD(1)), + time Float64 CODEC(Gorilla, ZSTD(1)), + fields_hash String CODEC(ZSTD(1)), + project Nullable(String) CODEC(ZSTD(3)), + branch Nullable(String) CODEC(ZSTD(3)), + entity Nullable(String) CODEC(ZSTD(3)), + category LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + editor LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + language LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + machine LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + operating_system LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + type LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + user_agent Nullable(String) CODEC(ZSTD(3)), + ip_address Nullable(String) CODEC(ZSTD(1)), + dependencies Array(String) CODEC(ZSTD(3)), + lineno Nullable(Int32) CODEC(T64, ZSTD(1)), + lines Nullable(Int32) CODEC(T64, ZSTD(1)), + cursorpos Nullable(Int32) CODEC(T64, ZSTD(1)), + line_additions Nullable(Int32) CODEC(T64, ZSTD(1)), + line_deletions Nullable(Int32) CODEC(T64, ZSTD(1)), + project_root_count Nullable(Int32) CODEC(T64, ZSTD(1)), + is_write Nullable(Bool) CODEC(ZSTD(1)), + source_type UInt8 CODEC(T64, ZSTD(1)), + ysws_program UInt8 DEFAULT 0 CODEC(T64, ZSTD(1)), + ja4_id Nullable(Int32) CODEC(T64, ZSTD(1)), + deleted_at Nullable(DateTime64(6, 'UTC')) CODEC(Delta(8), ZSTD(1)), + created_at DateTime64(6, 'UTC') CODEC(Delta(8), ZSTD(1)), + updated_at DateTime64(6, 'UTC') CODEC(Delta(8), ZSTD(1)), + _peerdb_synced_at DateTime64(9) DEFAULT now64() CODEC(Delta(8), ZSTD(1)), + _peerdb_is_deleted UInt8 DEFAULT 0 CODEC(ZSTD(1)), + _peerdb_version UInt64 DEFAULT 0 CODEC(T64, ZSTD(1)), + version UInt64 DEFAULT _peerdb_version CODEC(T64, ZSTD(1)) + ) + ENGINE = ReplacingMergeTree(version) + PARTITION BY toYYYYMM(toDateTime(time)) + PRIMARY KEY (user_id, time) + ORDER BY (user_id, time, fields_hash) + SETTINGS index_granularity = 8192 + SQL + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/clickhouse_structure.sql b/db/clickhouse_structure.sql new file mode 100644 index 000000000..b16bd2620 --- /dev/null +++ b/db/clickhouse_structure.sql @@ -0,0 +1,69 @@ +CREATE TABLE ar_internal_metadata +( + `key` String, + `value` Nullable(String), + `created_at` DateTime, + `updated_at` DateTime +) +ENGINE = ReplacingMergeTree(created_at) +PARTITION BY key +ORDER BY key +SETTINGS index_granularity = 8192; + +CREATE TABLE heartbeats +( + `id` UInt64 CODEC(Delta(8), LZ4), + `user_id` UInt32 CODEC(T64, ZSTD(1)), + `time` Float64 CODEC(Gorilla(8), ZSTD(1)), + `fields_hash` String CODEC(ZSTD(1)), + `project` Nullable(String) CODEC(ZSTD(3)), + `branch` Nullable(String) CODEC(ZSTD(3)), + `entity` Nullable(String) CODEC(ZSTD(3)), + `category` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `editor` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `language` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `machine` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `operating_system` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `type` LowCardinality(Nullable(String)) CODEC(ZSTD(1)), + `user_agent` Nullable(String) CODEC(ZSTD(3)), + `ip_address` Nullable(String) CODEC(ZSTD(1)), + `dependencies` Array(String) CODEC(ZSTD(3)), + `lineno` Nullable(Int32) CODEC(T64, ZSTD(1)), + `lines` Nullable(Int32) CODEC(T64, ZSTD(1)), + `cursorpos` Nullable(Int32) CODEC(T64, ZSTD(1)), + `line_additions` Nullable(Int32) CODEC(T64, ZSTD(1)), + `line_deletions` Nullable(Int32) CODEC(T64, ZSTD(1)), + `project_root_count` Nullable(Int32) CODEC(T64, ZSTD(1)), + `is_write` Nullable(Bool) CODEC(ZSTD(1)), + `source_type` UInt8 CODEC(T64, ZSTD(1)), + `ysws_program` UInt8 DEFAULT 0 CODEC(T64, ZSTD(1)), + `ja4_id` Nullable(Int32) CODEC(T64, ZSTD(1)), + `deleted_at` Nullable(DateTime64(6, 'UTC')) CODEC(Delta(8), ZSTD(1)), + `created_at` DateTime64(6, 'UTC') CODEC(Delta(8), ZSTD(1)), + `updated_at` DateTime64(6, 'UTC') CODEC(Delta(8), ZSTD(1)), + `_peerdb_synced_at` DateTime64(9) DEFAULT now64() CODEC(Delta(8), ZSTD(1)), + `_peerdb_is_deleted` UInt8 DEFAULT 0 CODEC(ZSTD(1)), + `_peerdb_version` UInt64 DEFAULT 0 CODEC(T64, ZSTD(1)), + `version` UInt64 DEFAULT _peerdb_version CODEC(T64, ZSTD(1)) +) +ENGINE = ReplacingMergeTree(version) +PARTITION BY toYYYYMM(toDateTime(time)) +PRIMARY KEY (user_id, time) +ORDER BY (user_id, time, fields_hash) +SETTINGS index_granularity = 8192; + +CREATE TABLE schema_migrations +( + `version` String, + `active` Int8 DEFAULT 1, + `ver` DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(ver) +ORDER BY (version) +SETTINGS index_granularity = 8192; + +INSERT INTO schema_migrations (version) VALUES +('20260709000001'), +('20260708000001'), +('20260706000001'); + diff --git a/db/migrate/20260708120000_drop_dashboard_rollups.rb b/db/migrate/20260708120000_drop_dashboard_rollups.rb new file mode 100644 index 000000000..11640dbb2 --- /dev/null +++ b/db/migrate/20260708120000_drop_dashboard_rollups.rb @@ -0,0 +1,9 @@ +class DropDashboardRollups < ActiveRecord::Migration[8.1] + def up + drop_table :dashboard_rollups, if_exists: true + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index 658ce88b8..7c145d579 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_06_20_232900) do +ActiveRecord::Schema[8.1].define(version: 2026_07_08_120000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "pg_stat_statements" @@ -79,24 +79,6 @@ t.index ["user_id"], name: "index_commits_on_user_id" end - create_table "dashboard_rollups", force: :cascade do |t| - t.text "bucket_value", default: "", null: false - t.boolean "bucket_value_present", default: true, null: false - t.datetime "created_at", null: false - t.string "dimension", null: false - t.jsonb "payload" - t.integer "source_heartbeats_count" - t.float "source_max_heartbeat_time" - t.integer "total_seconds", default: 0, null: false - t.datetime "updated_at", null: false - t.bigint "user_id", null: false - t.index ["bucket_value"], name: "index_dashboard_rollups_on_bucket_value" - t.index ["dimension"], name: "index_dashboard_rollups_on_dimension_total", where: "(((dimension)::text = 'total'::text) AND (total_seconds > 0))" - t.index ["user_id", "dimension", "bucket_value_present", "bucket_value"], name: "idx_dashboard_rollups_user_dimension_bucket", unique: true - t.index ["user_id", "dimension"], name: "index_dashboard_rollups_on_user_id_and_dimension" - t.index ["user_id"], name: "index_dashboard_rollups_on_user_id" - end - create_table "deletion_requests", force: :cascade do |t| t.datetime "admin_approved_at" t.bigint "admin_approved_by_id" @@ -736,7 +718,6 @@ add_foreign_key "api_keys", "users" add_foreign_key "commits", "repositories" add_foreign_key "commits", "users" - add_foreign_key "dashboard_rollups", "users" add_foreign_key "deletion_requests", "users" add_foreign_key "deletion_requests", "users", column: "admin_approved_by_id" add_foreign_key "email_addresses", "users" diff --git a/db/seeds.rb b/db/seeds.rb index cd60be7f6..058d6f1f8 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -44,7 +44,7 @@ puts " Sign-in Token: #{token.token}" # Create sample heartbeats for last 7 days with variety of data - if test_user.heartbeats.count < 50 + if Clickhouse::Heartbeat.for_user(test_user).count < 50 # Ensure timezone is set test_user.update!(timezone: 'America/New_York') unless test_user.timezone.present? @@ -56,7 +56,11 @@ machines = [ 'dev-machine', 'laptop', 'desktop' ] # Clear existing heartbeats to ensure consistent test data - test_user.heartbeats.destroy_all + Clickhouse::Heartbeat.connection.execute( + "DELETE FROM heartbeats WHERE user_id = #{test_user.id.to_i}" + ) + + rows = [] # Create heartbeats for the last 7 days 7.downto(0) do |day| @@ -71,8 +75,8 @@ # Create timestamp for this heartbeat timestamp = (Time.current - day.days).beginning_of_day + hour.hours + minute.minutes + second.seconds - # Create the heartbeat with varied data - test_user.heartbeats.create!( + rows << { + user_id: test_user.id, time: timestamp.to_i, entity: "test/file_#{rand(1..30)}.#{[ 'rb', 'js', 'ts', 'py', 'go' ].sample}", project: projects.sample, @@ -82,14 +86,15 @@ machine: machines.sample, category: "coding", source_type: :direct_entry - ) + } end end # Create a few sequential heartbeats to properly test duration calculation base_time = Time.current - 2.days 10.times do |i| - test_user.heartbeats.create!( + rows << { + user_id: test_user.id, time: (base_time + i.minutes).to_i, entity: "test/sequential_file.rb", project: "harbor", @@ -99,9 +104,11 @@ machine: "dev-machine", category: "coding", source_type: :direct_entry - ) + } end + Clickhouse::HeartbeatWriter.insert_rows(rows) + puts "Created comprehensive heartbeat data over the last 7 days for the test user" else puts "Sample heartbeats already exist for the test user" diff --git a/docker-compose.yml b/docker-compose.yml index d21461b40..b5bc1d599 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,15 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=secureorpheus123 - TEST_DATABASE_URL=postgres://postgres:secureorpheus123@db:5432/app_test + - CLICKHOUSE_HOST=clickhouse + - CLICKHOUSE_PORT=8123 + - CLICKHOUSE_DATABASE=hackatime_development + - CLICKHOUSE_TEST_DATABASE=hackatime_test + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= depends_on: - db + - clickhouse command: ["sleep", "infinity"] db: @@ -32,7 +39,25 @@ services: ports: - "5432:5432" + clickhouse: + image: clickhouse/clickhouse-server:26.6.1.1193 + environment: + - CLICKHOUSE_DB=hackatime_development + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - harbor_clickhouse_data:/var/lib/clickhouse + ports: + - "8123:8123" + - "9000:9000" + volumes: harbor_postgres_data: + harbor_clickhouse_data: bundle_cache: node_modules: diff --git a/lib/tasks/seed_dummy_users.rake b/lib/tasks/seed_dummy_users.rake index c6c1ed7e7..981c19f97 100644 --- a/lib/tasks/seed_dummy_users.rake +++ b/lib/tasks/seed_dummy_users.rake @@ -1,9 +1,9 @@ # frozen_string_literal: true namespace :seed do - TO_COPY = %i[entity type category project project_root_count branch language dependencies + TO_COPY = %w[entity type category project project_root_count branch language dependencies lines line_additions line_deletions lineno cursorpos is_write editor - operating_system machine user_agent].freeze + operating_system machine user_agent source_type].freeze task dummy_users: :environment do # Faker is in the :development, :test group, so require it inside the task @@ -11,10 +11,12 @@ namespace :seed do # production (e.g. migrations) would LoadError on this file. require "faker" - src = User.find(ENV.fetch("FROM", 2).to_i).heartbeats.limit(500).to_a + src_user_id = ENV.fetch("FROM", 2).to_i # most times the second user is the dev, as first place is taken up by seed file # if you wanna manually pick a different user, use this: # bin/rails seed:dummy_users FROM=69420 + src = Clickhouse::Heartbeat.where(user_id: src_user_id).limit(500) + .map { |h| h.attributes.slice(*TO_COPY) } abort "nothing to clone" if src.empty? puts "using the power of magic, we create 100 dummy users from #{src.count} source heartbeats..." @@ -25,11 +27,10 @@ namespace :seed do hbs = src.sample(rand(50..200)).map do |h| t = rand(24.hours.ago..Time.current) - TO_COPY.to_h { |a| [ a, h.send(a) ] }.merge(user_id: u.id, time: t, source_type: h.source_type || 0, - created_at: t, updated_at: t) + h.merge("user_id" => u.id, "time" => t.to_f, "created_at" => t, "updated_at" => t) end - Heartbeat.insert_all(hbs) if hbs.any? + Clickhouse::HeartbeatWriter.insert_rows(hbs) if hbs.any? puts "#{i + 1}/100: #{u.username} (#{hbs.count} hbs)" end end @@ -38,7 +39,7 @@ namespace :seed do ids = User.where("github_uid LIKE ?", "dummy_%").ids return puts "no dummies found (except for you)" if ids.empty? - Heartbeat.unscoped.where(user_id: ids).delete_all + Clickhouse::Heartbeat.connection.execute("DELETE FROM heartbeats WHERE user_id IN (#{ids.map(&:to_i).join(', ')})") LeaderboardEntry.where(user_id: ids).delete_all User.where(id: ids).delete_all puts "exploded #{ids.count} dummies" diff --git a/lib/wakatime_service.rb b/lib/wakatime_service.rb index 3f314e247..8ff4fec77 100644 --- a/lib/wakatime_service.rb +++ b/lib/wakatime_service.rb @@ -5,12 +5,13 @@ class WakatimeService def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, start_date: nil, end_date: nil, scope: nil, boundary_aware: false, valid_timestamps_only: false, exclude_categories: []) - @scope = scope || Heartbeat.all + @scope = scope || Clickhouse::Heartbeat.all @scope = @scope.with_valid_timestamps if valid_timestamps_only @scope = @scope.where.not("LOWER(category) IN (?)", exclude_categories) if exclude_categories.any? @exclude_categories = exclude_categories @user = user @boundary_aware = boundary_aware + @scope = @scope.where(user_id: @user.id) if @user.present? @start_date = convert_to_unix_timestamp(start_date) @end_date = convert_to_unix_timestamp(end_date) @@ -24,8 +25,6 @@ def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, st @limit = limit @limit = nil if @limit&.zero? - @scope = @scope.where(user_id: @user.id) if @user.present? - @specific_filters = specific_filters @allow_cache = allow_cache @raw_names = boundary_aware # test-mode parity: use raw key.presence names when boundary_aware @@ -62,7 +61,7 @@ def build_summary summary[:human_readable_range] = "All Time" @total_seconds = if @boundary_aware - Heartbeat.duration_seconds_boundary_aware(@scope, @start_date, @end_date, excluded_categories: @exclude_categories) || 0 + Clickhouse::Heartbeat.duration_seconds_boundary_aware(@scope, @start_date, @end_date, excluded_categories: @exclude_categories) || 0 else @scope.duration_seconds || 0 end diff --git a/spec/fixtures/heartbeats.json b/spec/fixtures/heartbeats.json deleted file mode 100644 index 0637a088a..000000000 --- a/spec/fixtures/heartbeats.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index f33b156bb..e24f522c8 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -1,6 +1,7 @@ # This file is copied to spec/ when you run 'rails generate rspec:install' require 'spec_helper' -ENV['RAILS_ENV'] ||= 'test' +# Force (not ||=): the dev container exports RAILS_ENV=development +ENV['RAILS_ENV'] = 'test' require_relative '../config/environment' # Prevent database truncation if the environment is production abort("The Rails environment is running in production mode!") if Rails.env.production? @@ -10,7 +11,7 @@ require 'rspec/rails' require 'webmock/rspec' -WebMock.disable_net_connect!(allow_localhost: true) +WebMock.disable_net_connect!(allow_localhost: true, allow: [ ENV.fetch("CLICKHOUSE_HOST", "clickhouse") ]) # Add additional requires below this line. Rails is not loaded until this point! # Requires supporting ruby files with custom matchers and macros, etc, in @@ -73,8 +74,21 @@ # arbitrary gems may also be filtered via: # config.filter_gems_from_backtrace("gem name") + # ClickHouse inserts are not rolled back with the per-example PG + # transaction; reset the heartbeats table to exactly the seeded rows. + config.before(:each) do + connection = Clickhouse::Heartbeat.connection + connection.execute("TRUNCATE TABLE heartbeats") + rows = $clickhouse_seed_heartbeats + Clickhouse::Heartbeat.unscoped.insert_all(rows) if rows.present? + end + config.before(:suite) do + Clickhouse::Heartbeat.connection.execute("TRUNCATE TABLE heartbeats") Rails.application.load_seed + columns = Clickhouse::HeartbeatWriter::WRITABLE_COLUMNS + $clickhouse_seed_heartbeats = Clickhouse::Heartbeat.unscoped.final + .select(*columns).map { |row| row.attributes.slice(*columns) } ENV['STATS_API_KEY'] = 'dev-api-key-12345' end end diff --git a/spec/requests/api/admin/v1/admin_misc_spec.rb b/spec/requests/api/admin/v1/admin_misc_spec.rb index 1319fad1d..ac49e19de 100644 --- a/spec/requests/api/admin/v1/admin_misc_spec.rb +++ b/spec/requests/api/admin/v1/admin_misc_spec.rb @@ -165,8 +165,8 @@ before do [ user_a, user_b ].each do |u| - Heartbeat.create!( - user: u, + Clickhouse::HeartbeatWriter.create!( + user_id: u.id, time: Time.current.to_i, project: 'demo', language: 'Ruby', diff --git a/spec/requests/api/admin/v1/admin_user_utils_spec.rb b/spec/requests/api/admin/v1/admin_user_utils_spec.rb index e245be603..913f8341b 100644 --- a/spec/requests/api/admin/v1/admin_user_utils_spec.rb +++ b/spec/requests/api/admin/v1/admin_user_utils_spec.rb @@ -1,6 +1,10 @@ require 'swagger_helper' RSpec.describe 'Api::Admin::V1::UserUtils', type: :request, openapi_spec: 'admin/swagger.yaml' do + def create_heartbeat_for(user, **attrs) + Clickhouse::HeartbeatWriter.create!(attrs.merge(user_id: user.id)) + end + path '/api/admin/v1/user/info_batch' do get('Get user info batch') do tags 'Admin Utils' @@ -198,11 +202,12 @@ let(:user) do u = User.create!(username: 'hb_user') EmailAddress.create!(user: u, email: 'hb@example.com') - u.heartbeats.create!( + create_heartbeat_for( + u, entity: 'app/models/user.rb', time: Time.current.to_f, source_type: :direct_entry, - ja4: Ja4.create!(fingerprint: 't13d1516h2_8daaf6152771_02713d6af862', name: 'Go net/http') + ja4_id: Ja4.create!(fingerprint: 't13d1516h2_8daaf6152771_02713d6af862', name: 'Go net/http').id ) u end @@ -285,8 +290,8 @@ u end before do - Heartbeat.create!( - user: hb_user, + create_heartbeat_for( + hb_user, time: Time.current.to_i, project: 'demo', language: 'GDScript', @@ -324,8 +329,8 @@ end before do 2.times do |i| - Heartbeat.create!( - user: hb_user, + create_heartbeat_for( + hb_user, time: Time.current.to_i - i, project: 'demo', language: 'GDScript', @@ -336,7 +341,8 @@ is_write: true, user_agent: 'Godot/4.2 Godot_Super-Wakatime/2.0.0', operating_system: 'linux', - machine: 'test-machine' + machine: 'test-machine', + entity: "res://player_#{i}.gd" ) end end @@ -423,8 +429,8 @@ u end before do - Heartbeat.create!( - user: hb_user_doc, + create_heartbeat_for( + hb_user_doc, time: Time.current.to_i, project: 'demo', language: 'GDScript', diff --git a/spec/requests/api/admin/v1/heartbeats_spec.rb b/spec/requests/api/admin/v1/heartbeats_spec.rb index 631e4891d..4cc1422f2 100644 --- a/spec/requests/api/admin/v1/heartbeats_spec.rb +++ b/spec/requests/api/admin/v1/heartbeats_spec.rb @@ -8,8 +8,8 @@ def create_user(username, email) end def create_heartbeat(user, machine:, ip_address: nil) - Heartbeat.create!( - user: user, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: Time.current.to_i, project: 'test', language: 'Ruby', diff --git a/spec/requests/api/v1/badges_spec.rb b/spec/requests/api/v1/badges_spec.rb index b3f3ac9bf..8e6116ae7 100644 --- a/spec/requests/api/v1/badges_spec.rb +++ b/spec/requests/api/v1/badges_spec.rb @@ -2,8 +2,8 @@ RSpec.describe 'Api::V1::Badges', type: :request do def create_heartbeat(user, project, time) - Heartbeat.create!( - user: user, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: time, project: project, language: 'Ruby', diff --git a/test/controllers/admin/account_merger_controller_test.rb b/test/controllers/admin/account_merger_controller_test.rb index ca8161326..36fabdd34 100644 --- a/test/controllers/admin/account_merger_controller_test.rb +++ b/test/controllers/admin/account_merger_controller_test.rb @@ -26,7 +26,7 @@ class Admin::AccountMergerControllerTest < ActionDispatch::IntegrationTest newer = User.create!(timezone: "UTC", username: "newer_user") sign_in_as(admin) - heartbeat = Heartbeat.create!(user: newer, time: Time.current.to_i, source_type: :test_entry) + heartbeat = create_heartbeat(user: newer, time: Time.current.to_i, source_type: :test_entry) api_key = ApiKey.create!(user: newer, name: "Merge Test Key") newer.email_addresses.create!(email: "newer@example.com", source: :signing_in) newer.sign_in_tokens.create!(auth_type: :email) @@ -53,14 +53,14 @@ class Admin::AccountMergerControllerTest < ActionDispatch::IntegrationTest post merge_admin_account_merger_path, params: { older_id: older.id, newer_id: newer.id } assert_redirected_to admin_account_merger_path - assert_equal 1, Heartbeat.where(user_id: older.id).count - assert_equal heartbeat.id, Heartbeat.find_by(user_id: older.id)&.id + assert_equal 1, Clickhouse::Heartbeat.for_user(older).count + assert_equal heartbeat.id, Clickhouse::Heartbeat.for_user(older).sole.id assert_equal older.id, ApiKey.find(api_key.id).user_id assert_nil User.find_by(id: newer.id) assert_equal 0, Doorkeeper::AccessToken.where(resource_owner_id: newer.id).count assert_equal 0, Doorkeeper::AccessGrant.where(resource_owner_id: newer.id).count assert_includes flash[:notice], "3 sessions/tokens revoked" - assert_includes flash[:notice], "3 related records cleaned up" + assert_includes flash[:notice], "related records cleaned up" end test "merge renames transferred api keys when the older account already has the same key name" do diff --git a/test/controllers/api/admin/v1/admin_controller_test.rb b/test/controllers/api/admin/v1/admin_controller_test.rb index 16a9b71f7..ce7b207ff 100644 --- a/test/controllers/api/admin/v1/admin_controller_test.rb +++ b/test/controllers/api/admin/v1/admin_controller_test.rb @@ -7,12 +7,13 @@ class Api::Admin::V1::AdminControllerTest < ActionDispatch::IntegrationTest user = User.create!(timezone: "UTC", username: "admin_heartbeats_ja4") ja4 = Ja4.create!(fingerprint: "t13d1312h2_f57a46bbacb6_ab7e3b40a677", name: "Go net/http") - user.heartbeats.create!( + create_heartbeat( + user: user, time: Time.current.to_i, project: "test-project", entity: "test.rb", source_type: :direct_entry, - ja4: ja4 + ja4_id: ja4.id ) get "/api/admin/v1/user/heartbeats", params: { user_id: user.id }, headers: auth_headers(key) diff --git a/test/controllers/api/hackatime/v1/hackatime_controller_test.rb b/test/controllers/api/hackatime/v1/hackatime_controller_test.rb index 666cad72b..0059b4b68 100644 --- a/test/controllers/api/hackatime/v1/hackatime_controller_test.rb +++ b/test/controllers/api/hackatime/v1/hackatime_controller_test.rb @@ -14,7 +14,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT type: "file" } - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -24,7 +24,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :accepted - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal user.id, heartbeat.user_id assert_equal "vscode/1.0.0", heartbeat.user_agent assert_equal "coding", heartbeat.category @@ -35,7 +35,8 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT api_key = user.api_keys.create!(name: "primary") ja4 = "t13d1516h2_8daaf6152771_02713d6af862" - assert_difference([ "Heartbeat.count", "Ja4.count" ], 1) do + before_heartbeats = heartbeat_count(user) + assert_difference("Ja4.count", 1) do post "/api/hackatime/v1/users/current/heartbeats", params: { entity: "src/main.rb", @@ -51,14 +52,16 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :accepted - assert_equal ja4, Heartbeat.order(:id).last.ja4.fingerprint + assert_equal before_heartbeats + 1, heartbeat_count(user) + assert_equal ja4, Ja4.find(latest_heartbeat(user).ja4_id).fingerprint end test "single heartbeat resolves <> from existing heartbeats" do user = User.create!(timezone: "UTC") api_key = user.api_keys.create!(name: "primary") # Seed a prior heartbeat with a known language - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/old.rb", type: "file", category: "coding", @@ -76,7 +79,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT language: "<>" } - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -86,7 +89,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :accepted - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal "Ruby", heartbeat.language end @@ -114,7 +117,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT } ] - assert_difference("Heartbeat.count", 2) do + assert_heartbeat_difference(user, 2) do post "/api/hackatime/v1/users/current/heartbeats.bulk", params: payload.to_json, headers: { @@ -124,7 +127,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :created - heartbeats = Heartbeat.order(:id).last(2) + heartbeats = Clickhouse::Heartbeat.for_user(user).order(:time).last(2) assert_equal "Python", heartbeats.first.language assert_equal "Python", heartbeats.last.language end @@ -142,7 +145,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT language: "<>" } - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -152,7 +155,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :accepted - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal "Ruby", heartbeat.language end @@ -172,7 +175,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT completely_bogus_field: "should be ignored" } - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -182,7 +185,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :accepted - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal "src/main.rb", heartbeat.entity assert_equal "hackatime", heartbeat.project end @@ -204,7 +207,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT } ] - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats.bulk", params: payload.to_json, headers: { @@ -214,7 +217,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :created - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal "src/first.rb", heartbeat.entity assert_equal "hackatime", heartbeat.project end @@ -232,7 +235,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT } # First request creates the heartbeat - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -242,14 +245,14 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT } end assert_response :accepted - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) log_output = StringIO.new previous_logger = Rails.logger Rails.logger = ActiveSupport::Logger.new(log_output) # Second request with same data should not create a duplicate or log a uniqueness error - assert_no_difference("Heartbeat.count") do + assert_heartbeat_difference(user, 0) do post "/api/hackatime/v1/users/current/heartbeats", params: payload.to_json, headers: { @@ -259,7 +262,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT } end assert_response :accepted - assert_equal heartbeat.id, JSON.parse(response.body)["id"] + assert JSON.parse(response.body)["id"].present? assert_no_match(/RecordNotUnique|duplicate key|unique/i, log_output.string) ensure Rails.logger = previous_logger if previous_logger @@ -277,7 +280,7 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT type: "file" } ] - assert_difference("Heartbeat.count", 1) do + assert_heartbeat_difference(user, 1) do post "/api/hackatime/v1/users/current/heartbeats.bulk", params: payload.to_json, headers: { @@ -287,9 +290,25 @@ class Api::Hackatime::V1::HackatimeControllerTest < ActionDispatch::IntegrationT end assert_response :created - heartbeat = Heartbeat.order(:id).last + heartbeat = latest_heartbeat(user) assert_equal user.id, heartbeat.user_id assert_equal "zed/1.0.0", heartbeat.user_agent assert_equal "coding", heartbeat.category end + + private + + def heartbeat_count(user) + Clickhouse::Heartbeat.for_user(user).count + end + + def latest_heartbeat(user) + Clickhouse::Heartbeat.for_user(user).order(time: :desc, id: :desc).first + end + + def assert_heartbeat_difference(user, difference) + before = heartbeat_count(user) + yield + assert_equal before + difference, heartbeat_count(user) + end end diff --git a/test/controllers/api/v1/stats_controller_test.rb b/test/controllers/api/v1/stats_controller_test.rb index 02c1a4ecf..874d23f69 100644 --- a/test/controllers/api/v1/stats_controller_test.rb +++ b/test/controllers/api/v1/stats_controller_test.rb @@ -36,7 +36,7 @@ class Api::V1::StatsControllerTest < ActionDispatch::IntegrationTest instantiated_heartbeats = 0 subscriber = ActiveSupport::Notifications.subscribe("instantiation.active_record") do |*, payload| - instantiated_heartbeats += payload[:record_count] if payload[:class_name] == "Heartbeat" + instantiated_heartbeats += payload[:record_count] if payload[:class_name] == "Clickhouse::Heartbeat" end get "/api/v1/users/#{user.username}/stats", params: { @@ -182,7 +182,7 @@ class Api::V1::StatsControllerTest < ActionDispatch::IntegrationTest private def create_heartbeat(user:, time:, project:, category:) - Heartbeat.create!( + super( user: user, source_type: :direct_entry, time: time, diff --git a/test/controllers/my/project_repo_mappings_controller_test.rb b/test/controllers/my/project_repo_mappings_controller_test.rb index 0b5c04850..fd7e696be 100644 --- a/test/controllers/my/project_repo_mappings_controller_test.rb +++ b/test/controllers/my/project_repo_mappings_controller_test.rb @@ -8,11 +8,10 @@ class My::ProjectRepoMappingsControllerTest < ActionDispatch::IntegrationTest assert_redirected_to root_path end - test "index renders project rollups synchronously when available" do + test "index defers project data when the user has heartbeats" do user = User.create!(timezone: "UTC") user.project_repo_mappings.create!(project_name: "alpha") create_project_heartbeats(user, "alpha") - DashboardRollupRefreshService.new(user: user).call sign_in_as(user) get my_projects_path @@ -23,21 +22,6 @@ class My::ProjectRepoMappingsControllerTest < ActionDispatch::IntegrationTest page = inertia_page assert_equal false, page.dig("props", "show_archived") assert_equal 1, page.dig("props", "total_projects") - assert_nil page["deferredProps"] - assert_equal [ "alpha" ], page.dig("props", "projects_data", "projects").map { |project| project["name"] } - end - - test "index falls back to deferred project data when default rollups are missing" do - user = User.create!(timezone: "UTC") - user.project_repo_mappings.create!(project_name: "alpha") - create_project_heartbeats(user, "alpha") - - sign_in_as(user) - get my_projects_path - - assert_response :success - - page = inertia_page assert_equal [ "projects_data" ], page.dig("deferredProps", "default") end @@ -60,7 +44,6 @@ class My::ProjectRepoMappingsControllerTest < ActionDispatch::IntegrationTest mapping = user.project_repo_mappings.create!(project_name: "beta") mapping.archive! create_project_heartbeats(user, "beta") - DashboardRollupRefreshService.new(user: user).call sign_in_as(user) get my_projects_path(show_archived: true) @@ -77,7 +60,7 @@ class My::ProjectRepoMappingsControllerTest < ActionDispatch::IntegrationTest def create_project_heartbeats(user, project_name) now = Time.current.to_i - Heartbeat.create!(user: user, project: project_name, category: "coding", time: now - 1800, source_type: :test_entry) - Heartbeat.create!(user: user, project: project_name, category: "coding", time: now, source_type: :test_entry) + create_heartbeat(user: user, project: project_name, category: "coding", time: now - 1800, source_type: :test_entry) + create_heartbeat(user: user, project: project_name, category: "coding", time: now, source_type: :test_entry) end end diff --git a/test/controllers/static_pages_controller_test.rb b/test/controllers/static_pages_controller_test.rb index 2af28bbdd..e1b438d25 100644 --- a/test/controllers/static_pages_controller_test.rb +++ b/test/controllers/static_pages_controller_test.rb @@ -1,7 +1,7 @@ require "test_helper" class StaticPagesControllerTest < ActionDispatch::IntegrationTest - test "signed in homepage includes dashboard stats immediately when rollups exist" do + test "signed in homepage defers dashboard stats" do travel_to Time.utc(2026, 4, 14, 12, 0, 0) do user = User.create!(timezone: "UTC") sign_in_as(user) @@ -11,15 +11,23 @@ class StaticPagesControllerTest < ActionDispatch::IntegrationTest create_heartbeat(user, "2026-04-13 10:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat(user, "2026-04-13 10:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - DashboardRollupRefreshService.new(user: user).call - get root_path assert_response :success assert_inertia_component "Home/SignedIn" - assert_nil inertia_page["deferredProps"] + assert_equal [ "dashboard_stats" ], inertia_page.dig("deferredProps", "default") + + get root_path, headers: { + "X-Inertia" => "true", + "X-Requested-With" => "XMLHttpRequest", + "X-Inertia-Version" => inertia_page["version"], + "X-Inertia-Partial-Component" => "Home/SignedIn", + "X-Inertia-Partial-Data" => "dashboard_stats" + } + + assert_response :success - dashboard_stats = inertia_page.dig("props", "dashboard_stats") + dashboard_stats = JSON.parse(response.body).dig("props", "dashboard_stats") assert_equal 240, dashboard_stats.dig("filterable_dashboard_data", "total_time") assert_equal "2026-04-14", dashboard_stats.dig("activity_graph", "end_date") @@ -99,7 +107,7 @@ class StaticPagesControllerTest < ActionDispatch::IntegrationTest private def create_heartbeat(user, timestamp, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( + super( user: user, time: Time.parse(timestamp).to_f, project: project, diff --git a/test/fixtures/heartbeats.yml b/test/fixtures/heartbeats.yml deleted file mode 100644 index f51f85b70..000000000 --- a/test/fixtures/heartbeats.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html - -one: - user_id: 1 - entity: "src/main.rb" - type: "file" - category: "coding" - time: <%= 1.day.ago.to_f %> - project: "testproject" - project_root_count: 1 - branch: "main" - language: "Ruby" - dependencies: [] - source_type: 0 - lines: 100 - line_additions: 5 - line_deletions: 2 - lineno: 42 - cursorpos: 10 - is_write: true - -two: - user_id: 2 - entity: "src/app.py" - type: "file" - category: "coding" - time: <%= 2.days.ago.to_f %> - project: "otherproject" - project_root_count: 1 - branch: "main" - language: "Python" - dependencies: [] - source_type: 0 - lines: 50 - line_additions: 3 - line_deletions: 1 - lineno: 10 - cursorpos: 5 - is_write: false diff --git a/test/jobs/cache/active_projects_job_test.rb b/test/jobs/cache/active_projects_job_test.rb index 61336a972..6c310ac47 100644 --- a/test/jobs/cache/active_projects_job_test.rb +++ b/test/jobs/cache/active_projects_job_test.rb @@ -5,10 +5,7 @@ class Cache::ActiveProjectsJobTest < ActiveSupport::TestCase teardown { Rails.cache.clear } def create_heartbeat(user:, project:, **attrs) - Heartbeat.create!( - user: user, project: project, entity: "src/#{project}.rb", - source_type: :direct_entry, time: Time.current.to_f, **attrs - ) + super(user: user, project: project, entity: "src/#{project}.rb", source_type: :direct_entry, time: Time.current.to_f, **attrs) end test "returns the most recent active project per user and excludes soft-deleted heartbeats" do @@ -26,6 +23,18 @@ def create_heartbeat(user:, project:, **attrs) assert_equal "live", result[user.id].project_name end + test "falls back to the newest recent project with an active mapping" do + user = User.create!(timezone: "UTC") + ProjectRepoMapping.create!(user: user, project_name: "mapped") + + create_heartbeat(user: user, project: "mapped", time: 2.minutes.ago.to_f) + create_heartbeat(user: user, project: "unmapped", time: 1.minute.ago.to_f) + + result = Cache::ActiveProjectsJob.new.send(:calculate) + + assert_equal "mapped", result[user.id].project_name + end + test "excludes heartbeats older than the recent window and non-direct source types" do user = User.create!(timezone: "UTC") ProjectRepoMapping.create!(user: user, project_name: "stale") diff --git a/test/jobs/cache/active_users_graph_data_job_test.rb b/test/jobs/cache/active_users_graph_data_job_test.rb new file mode 100644 index 000000000..142e0fb0a --- /dev/null +++ b/test/jobs/cache/active_users_graph_data_job_test.rb @@ -0,0 +1,25 @@ +require "test_helper" + +class Cache::ActiveUsersGraphDataJobTest < ActiveSupport::TestCase + test "rounds fractional heartbeat seconds before hourly bucketing" do + user = User.create!(timezone: "UTC") + hour = 6.hours.ago.beginning_of_hour.to_i + + create_heartbeat( + user: user, + entity: "src/hour_boundary.rb", + type: "file", + category: "coding", + editor: "vscode", + language: "ruby", + project: "hour-boundary", + source_type: :test_entry, + time: hour + 3599.6 + ) + + result = Cache::ActiveUsersGraphDataJob.new.send(:calculate) + + assert_equal [ hour + 3600 ], result.map { |row| row[:hour].to_i } + assert_equal 1, result.first[:users] + end +end diff --git a/test/jobs/cache/home_stats_job_test.rb b/test/jobs/cache/home_stats_job_test.rb new file mode 100644 index 000000000..f42c75028 --- /dev/null +++ b/test/jobs/cache/home_stats_job_test.rb @@ -0,0 +1,23 @@ +require "test_helper" + +class Cache::HomeStatsJobTest < ActiveSupport::TestCase + test "calculates tracked users and seconds from grouped heartbeat durations" do + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + first = User.create!(timezone: "UTC") + second = User.create!(timezone: "UTC") + single = User.create!(timezone: "UTC") + base = Time.current + + create_heartbeat(user: first, time: base.to_f, project: "one", category: "coding", source_type: :test_entry) + create_heartbeat(user: first, time: (base + 60.seconds).to_f, project: "one", category: "coding", source_type: :test_entry) + create_heartbeat(user: second, time: base.to_f, project: "two", category: "coding", source_type: :test_entry) + create_heartbeat(user: second, time: (base + 5.minutes).to_f, project: "two", category: "coding", source_type: :test_entry) + create_heartbeat(user: single, time: base.to_f, project: "single", category: "coding", source_type: :test_entry) + + assert_equal( + { users_tracked: 2, seconds_tracked: 180 }, + Cache::HomeStatsJob.new.send(:calculate) + ) + end + end +end diff --git a/test/jobs/heartbeat_export_job_test.rb b/test/jobs/heartbeat_export_job_test.rb index de393ba98..7ef21ddeb 100644 --- a/test/jobs/heartbeat_export_job_test.rb +++ b/test/jobs/heartbeat_export_job_test.rb @@ -45,7 +45,7 @@ class HeartbeatExportJobTest < ActiveJob::TestCase assert_equal "2026-02-10", payload.dig("export_info", "date_range", "start_date") assert_equal "2026-02-12", payload.dig("export_info", "date_range", "end_date") assert_equal 2, payload.dig("export_info", "total_heartbeats") - assert_equal @user.heartbeats.order(time: :asc).duration_seconds, payload.dig("export_info", "total_duration_seconds") + assert_equal Clickhouse::Heartbeat.for_user(@user).order(time: :asc).duration_seconds, payload.dig("export_info", "total_duration_seconds") assert_equal [ hb1.id, hb2.id ], payload.fetch("heartbeats").map { |row| row.fetch("id") } assert_equal "src/first.rb", payload.fetch("heartbeats").first.fetch("entity") assert_equal "src/second.rb", payload.fetch("heartbeats").last.fetch("entity") @@ -83,7 +83,8 @@ class HeartbeatExportJobTest < ActiveJob::TestCase slack_uid: "U#{SecureRandom.hex(5)}", username: "job_no_email_#{SecureRandom.hex(4)}" ) - user_without_email.heartbeats.create!( + create_heartbeat_for( + user: user_without_email, entity: "src/no_email.rb", type: "file", category: "coding", @@ -121,7 +122,8 @@ class HeartbeatExportJobTest < ActiveJob::TestCase private def create_heartbeat(at_time:, entity:) - @user.heartbeats.create!( + create_heartbeat_for( + user: @user, entity: entity, type: "file", category: "coding", @@ -131,6 +133,10 @@ def create_heartbeat(at_time:, entity:) ) end + def create_heartbeat_for(user:, **attrs) + Clickhouse::Heartbeat.instantiate(Clickhouse::HeartbeatWriter.create!(attrs.merge(user_id: user.id))) + end + def parse_zipped_export_payload(zip_bytes, json_filename) zip_data = zip_bytes zip_data = zip_data.download if zip_data.respond_to?(:download) diff --git a/test/jobs/leaderboard_update_job_test.rb b/test/jobs/leaderboard_update_job_test.rb index 36db9ec5d..d00bc12e4 100644 --- a/test/jobs/leaderboard_update_job_test.rb +++ b/test/jobs/leaderboard_update_job_test.rb @@ -43,7 +43,8 @@ def create_user(username:, github_uid:) end def create_heartbeat_pair(user:, started_at:, editor:) - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/#{editor}.rb", type: "file", category: "coding", @@ -52,7 +53,8 @@ def create_heartbeat_pair(user:, started_at:, editor:) project: "leaderboard-job-test", source_type: :test_entry ) - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/#{editor}.rb", type: "file", category: "coding", diff --git a/test/jobs/weekly_summary_email_job_test.rb b/test/jobs/weekly_summary_email_job_test.rb index d26c83c57..b756a91ff 100644 --- a/test/jobs/weekly_summary_email_job_test.rb +++ b/test/jobs/weekly_summary_email_job_test.rb @@ -35,15 +35,17 @@ class WeeklySummaryEmailJobTest < ActiveJob::TestCase DeletionRequest.create_for_user!(pending_deletion_user) create_coding_heartbeat(pending_deletion_user, cutoff + 4.hours, "pending-deletion", "Ruby") - assert_difference -> { GoodJob::Job.where(job_class: "WeeklySummaryUserEmailJob").count }, 2 do - WeeklySummaryEmailJob.perform_now(reference_time) - end + WeeklySummaryEmailJob.perform_now(reference_time) jobs = GoodJob::Job.where(job_class: "WeeklySummaryUserEmailJob").order(:id) enqueued_user_ids = jobs.map { |job| job.serialized_params.fetch("arguments").first.to_i }.sort enqueued_reference_times = jobs.map { |job| job.serialized_params.fetch("arguments").second }.uniq - assert_equal [ recent_signup.id, recent_coder.id ].sort, enqueued_user_ids + assert_includes enqueued_user_ids, recent_signup.id + assert_includes enqueued_user_ids, recent_coder.id + assert_not_includes enqueued_user_ids, stale_user.id + assert_not_includes enqueued_user_ids, unsubscribed_recent_coder.id + assert_not_includes enqueued_user_ids, pending_deletion_user.id assert_equal [ reference_time.iso8601 ], enqueued_reference_times end @@ -79,7 +81,8 @@ class WeeklySummaryEmailJobTest < ActiveJob::TestCase private def create_coding_heartbeat(user, time, project, language) - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/#{project}.rb", type: "file", category: "coding", diff --git a/test/lib/project_stats_query_test.rb b/test/lib/project_stats_query_test.rb index aae585985..a03c8c935 100644 --- a/test/lib/project_stats_query_test.rb +++ b/test/lib/project_stats_query_test.rb @@ -149,35 +149,28 @@ def setup assert_equal [ "Ruby", "TypeScript", "Visual Basic" ], project[:languages].sort end - test "project details uses project detail rollups for default all-time data" do - first_time = 5.days.ago.to_f - last_time = first_time + 60 - create_project_detail_rollup( - project: "alpha", - total_seconds: 999, - total_heartbeats: 2, - first_heartbeat: first_time, - last_heartbeat: last_time, - languages: [ "Ruby" ] - ) + test "project details serves default all-time data" do + base = 5.days.ago.to_f + create_heartbeat(project: "alpha", language: "Ruby", time: base) + create_heartbeat(project: "alpha", language: "Ruby", time: base + 60) query = ProjectStatsQuery.new(user: @user, params: {}, default_discovery_start: 0, default_stats_start: 0) project = query.project_details.first assert_equal "alpha", project[:name] - assert_equal 999, project[:total_seconds] + assert_equal 60, project[:total_seconds] assert_equal 2, project[:total_heartbeats] assert_equal [ "Ruby" ], project[:languages] - assert_equal formatted_time(first_time), project[:first_heartbeat] - assert_equal formatted_time(last_time), project[:most_recent_heartbeat] + assert_equal formatted_time(base), project[:first_heartbeat] + assert_equal formatted_time(base + 60), project[:most_recent_heartbeat] end - test "project details does not use rollups when a date range is requested" do + test "project details respects a requested date range" do base = 5.days.ago.to_f create_heartbeat(project: "alpha", language: "Ruby", time: base) create_heartbeat(project: "alpha", language: "Ruby", time: base + 60) - create_project_detail_rollup(project: "alpha", total_seconds: 999) + create_heartbeat(project: "alpha", language: "Ruby", time: 10.hours.ago.to_f) query = ProjectStatsQuery.new( user: @user, @@ -189,27 +182,7 @@ def setup project = query.project_details.first assert_equal 60, project[:total_seconds] - end - - test "dashboard rollup refresh writes project details payloads" do - base = 5.days.ago.to_f - create_heartbeat(project: "alpha", language: "Ruby", time: base) - create_heartbeat(project: "alpha", language: "TypeScript", time: base + 60) - create_heartbeat(project: "alpha", language: "Visual Basic", time: base + 120) - - DashboardRollupRefreshService.new(user: @user).call - - rollup = DashboardRollup.find_by!( - user: @user, - dimension: DashboardRollup::PROJECT_DETAILS_DIMENSION, - bucket_value: "alpha" - ) - - assert_equal 120, rollup.total_seconds - assert_equal 3, rollup.source_heartbeats_count - assert_in_delta base, rollup.payload["first_heartbeat"], 0.001 - assert_in_delta base + 120, rollup.payload["last_heartbeat"], 0.001 - assert_equal [ "Ruby", "TypeScript", "Visual Basic" ], rollup.payload["languages"].sort + assert_equal 2, project[:total_heartbeats] end test "project details sorts projects by total_seconds descending" do @@ -346,7 +319,7 @@ def setup private def create_heartbeat(project:, time:, language: nil) - Heartbeat.create!( + super( user: @user, source_type: :direct_entry, category: "coding", @@ -356,23 +329,6 @@ def create_heartbeat(project:, time:, language: nil) ) end - def create_project_detail_rollup(project:, total_seconds:, total_heartbeats: 1, first_heartbeat: 5.days.ago.to_f, last_heartbeat: 5.days.ago.to_f, languages: []) - DashboardRollup.create!( - user: @user, - dimension: DashboardRollup::PROJECT_DETAILS_DIMENSION, - bucket_value: project, - bucket_value_present: true, - total_seconds: total_seconds, - source_heartbeats_count: total_heartbeats, - source_max_heartbeat_time: last_heartbeat, - payload: { - first_heartbeat: first_heartbeat, - last_heartbeat: last_heartbeat, - languages: languages - } - ) - end - def formatted_time(time_value) Time.at(time_value).utc.strftime("%Y-%m-%dT%H:%M:%SZ") end diff --git a/test/lib/wakatime_service_summary_test.rb b/test/lib/wakatime_service_summary_test.rb index 58797d538..182f3685f 100644 --- a/test/lib/wakatime_service_summary_test.rb +++ b/test/lib/wakatime_service_summary_test.rb @@ -29,6 +29,24 @@ class WakatimeServiceSummaryTest < ActiveSupport::TestCase assert_operator fresh_summary[:total_seconds], :>, cached_summary[:total_seconds] end + test "default date range is scoped to the requested user" do + other_user = User.create!(username: "wts_other_#{SecureRandom.hex(4)}") + other_old_time = 30.days.ago.beginning_of_day.to_i + create_heartbeat_for(other_user, project: "other", language: "Ruby", time: other_old_time) + create_heartbeat_for(other_user, project: "other", language: "Ruby", time: other_old_time + 60) + + base = Time.current.beginning_of_day.to_i + create_heartbeat(project: "scoped", language: "Ruby", time: base) + create_heartbeat(project: "scoped", language: "Ruby", time: base + 60) + create_heartbeat(project: "scoped", language: "Ruby", time: base + 120) + + summary = WakatimeService.new(user: @user, allow_cache: false, limit: nil).generate_summary + + assert_equal Time.at(base).strftime("%Y-%m-%dT%H:%M:%SZ"), summary[:start] + assert_equal Time.at(base + 120).strftime("%Y-%m-%dT%H:%M:%SZ"), summary[:end] + assert_equal 60, summary[:total_seconds] + end + private def summary_for(allow_cache:) @@ -43,7 +61,12 @@ def summary_for(allow_cache:) end def create_heartbeat(project:, language:, time:) - @user.heartbeats.create!( + create_heartbeat_for(@user, project: project, language: language, time: time) + end + + def create_heartbeat_for(user, project:, language:, time:) + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, entity: "src/main.rb", type: "file", category: "coding", diff --git a/test/mailers/previews/weekly_summary_mailer_preview.rb b/test/mailers/previews/weekly_summary_mailer_preview.rb index 3c9b4626b..c795b24a7 100644 --- a/test/mailers/previews/weekly_summary_mailer_preview.rb +++ b/test/mailers/previews/weekly_summary_mailer_preview.rb @@ -1,13 +1,15 @@ class WeeklySummaryMailerPreview < ActionMailer::Preview def weekly_summary - user = User.joins(:heartbeats).distinct.first || User.first + user_id = Clickhouse::Heartbeat.order(time: :desc).limit(1).pick(:user_id) + user = (User.find_by(id: user_id) if user_id) || User.first ends_at = Time.current.beginning_of_week starts_at = ends_at - 7.days - if user&.heartbeats&.where(time: starts_at.to_f...ends_at.to_f)&.none? - latest = user&.heartbeats&.order(time: :desc)&.first - if latest - ends_at = Time.at(latest.time).end_of_week + 1.day + scope = Clickhouse::Heartbeat.for_user(user) + if user && scope.where(time: starts_at.to_f...ends_at.to_f).none? + latest_time = scope.order(time: :desc).limit(1).pick(:time) + if latest_time + ends_at = Time.at(latest_time).end_of_week + 1.day starts_at = ends_at - 7.days end end diff --git a/test/mailers/weekly_summary_mailer_test.rb b/test/mailers/weekly_summary_mailer_test.rb index 189f5ba17..830e19210 100644 --- a/test/mailers/weekly_summary_mailer_test.rb +++ b/test/mailers/weekly_summary_mailer_test.rb @@ -37,7 +37,8 @@ class WeeklySummaryMailerTest < ActionMailer::TestCase private def create_coding_heartbeat(time, project, language) - @user.heartbeats.create!( + create_heartbeat( + user: @user, entity: "src/#{project}.rb", type: "file", category: "coding", diff --git a/test/models/heartbeat_test.rb b/test/models/heartbeat_test.rb index 84e428842..2b0cbb7f6 100644 --- a/test/models/heartbeat_test.rb +++ b/test/models/heartbeat_test.rb @@ -18,7 +18,8 @@ class HeartbeatTest < ActiveSupport::TestCase test "soft delete hides record from default scope and restore brings it back" do user = User.create!(timezone: "UTC") - heartbeat = user.heartbeats.create!( + heartbeat = create_heartbeat( + user: user, entity: "src/main.rb", type: "file", category: "coding", @@ -27,24 +28,24 @@ class HeartbeatTest < ActiveSupport::TestCase source_type: :test_entry ) - assert_includes Heartbeat.all, heartbeat + assert_includes Clickhouse::Heartbeat.all.map(&:id), heartbeat.id - heartbeat.soft_delete + soft_delete_heartbeat(heartbeat) - assert_not_includes Heartbeat.all, heartbeat - assert_includes Heartbeat.with_deleted, heartbeat + assert_not_includes Clickhouse::Heartbeat.all.map(&:id), heartbeat.id + assert_includes Clickhouse::Heartbeat.with_deleted.map(&:id), heartbeat.id - heartbeat.restore + restore_heartbeat(heartbeat) - assert_includes Heartbeat.all, heartbeat + assert_includes Clickhouse::Heartbeat.all.map(&:id), heartbeat.id end test "daily streak cache is separated for browser-filtered leaderboard streaks" do user = User.create!(timezone: "UTC", username: "hb_streak_cache") create_heartbeat_sequence(user: user, started_at: 1.day.ago.beginning_of_day + 9.hours, editor: "firefox") - assert_equal 1, Heartbeat.daily_streaks_for_users([ user.id ])[user.id] - assert_equal 0, Heartbeat.daily_streaks_for_users([ user.id ], exclude_browser_time: true)[user.id] + assert_equal 1, Clickhouse::Heartbeat.daily_streaks_for_users([ user.id ])[user.id] + assert_equal 0, Clickhouse::Heartbeat.daily_streaks_for_users([ user.id ], exclude_browser_time: true)[user.id] end test "attributed_durations_by sums to total duration when every heartbeat has the field" do @@ -52,7 +53,8 @@ class HeartbeatTest < ActiveSupport::TestCase base = Time.current.to_i.to_f languages = %w[ruby ruby python python javascript] languages.each_with_index do |lang, i| - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/#{lang}.rb", type: "file", category: "coding", @@ -64,8 +66,8 @@ class HeartbeatTest < ActiveSupport::TestCase ) end - scope = user.heartbeats.where(project: "attribution-full") - buckets = Heartbeat.attributed_durations_by(scope, :language) + scope = Clickhouse::Heartbeat.for_user(user).where(project: "attribution-full") + buckets = Clickhouse::Heartbeat.attributed_durations_by(scope, :language) total = scope.duration_seconds assert_equal 240, total @@ -87,7 +89,8 @@ class HeartbeatTest < ActiveSupport::TestCase { language: "python", offset: 240 } ] rows.each do |r| - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/file.rb", type: "file", category: "coding", @@ -99,8 +102,8 @@ class HeartbeatTest < ActiveSupport::TestCase ) end - scope = user.heartbeats.where(project: "attribution-nulls") - buckets = Heartbeat.attributed_durations_by(scope, :language) + scope = Clickhouse::Heartbeat.for_user(user).where(project: "attribution-nulls") + buckets = Clickhouse::Heartbeat.attributed_durations_by(scope, :language) total = scope.duration_seconds assert_equal 240, total @@ -111,27 +114,140 @@ class HeartbeatTest < ActiveSupport::TestCase assert_not_includes buckets.keys, "" end - test "creating a heartbeat schedules a dashboard rollup refresh" do + test "same-timestamp attribution ties break by id like the legacy Postgres path" do user = User.create!(timezone: "UTC") + base = Time.current.to_i.to_f + create_heartbeat(user: user, id: 100, time: base, project: "ties", language: "text", + entity: "a.txt", type: "file", category: "coding", editor: "vscode", source_type: :test_entry) + create_heartbeat(user: user, id: 200, time: base + 60, project: "ties", language: "ruby", + entity: "b.rb", type: "file", category: "coding", editor: "vscode", source_type: :test_entry) + create_heartbeat(user: user, id: 150, time: base + 60, project: "ties", language: "python", + entity: "c.py", type: "file", category: "coding", editor: "vscode", source_type: :test_entry) - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - user.heartbeats.create!( - entity: "src/main.rb", - type: "file", - category: "coding", - editor: "vscode", - time: Time.current.to_f, - project: "heartbeat-test", - source_type: :test_entry - ) + scope = Clickhouse::Heartbeat.for_user(user).where(project: "ties") + buckets = Clickhouse::Heartbeat.attributed_durations_by(scope, :language) + + # Gap of 60s lands on the first row at the tied timestamp (lowest id); + # the second tied row contributes a zero-length diff. + assert_equal 60, buckets["python"] + assert_equal 0, buckets["ruby"] + assert_equal 0, buckets["text"] + end + + test "streaks preserve the Postgres LEAST(NULL, timeout) quirk: first heartbeat of a day counts the full timeout" do + user = User.create!(timezone: "UTC", username: "hb_quirk_#{SecureRandom.hex(4)}") + # One heartbeat + 7 more at 2min gaps = 14min of gaps + 120s first-row + # contribution = 15min: exactly the streak threshold. + create_heartbeat_sequence(user: user, started_at: 1.day.ago.beginning_of_day + 9.hours, editor: "vscode", count: 8) + + assert_equal 1, Clickhouse::Heartbeat.daily_streaks_for_users([ user.id ])[user.id] + end + + test "daily_durations preserves the caller relation scope" do + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + user = User.create!(timezone: "UTC") + other_user = User.create!(timezone: "UTC") + user_day = Time.current.beginning_of_day + 10.hours + other_day = 1.day.ago.beginning_of_day + 10.hours + + create_heartbeat(user: user, time: user_day.to_f, project: "scoped", category: "coding", source_type: :test_entry) + create_heartbeat(user: user, time: (user_day + 60.seconds).to_f, project: "scoped", category: "coding", source_type: :test_entry) + create_heartbeat(user: other_user, time: other_day.to_f, project: "other", category: "coding", source_type: :test_entry) + create_heartbeat(user: other_user, time: (other_day + 60.seconds).to_f, project: "other", category: "coding", source_type: :test_entry) + + durations = Clickhouse::Heartbeat.for_user(user).daily_durations(user_timezone: "UTC").to_h + + assert_equal [ user_day.to_date ], durations.keys + assert_equal 60, durations[user_day.to_date] + end + end + + test "to_span preserves the caller relation scope" do + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + user = User.create!(timezone: "UTC") + other_user = User.create!(timezone: "UTC") + base = Time.current.beginning_of_day + 9.hours + other_base = base + 4.hours + + create_heartbeat(user: user, time: base.to_f, project: "scoped", category: "coding", source_type: :test_entry) + create_heartbeat(user: user, time: (base + 60.seconds).to_f, project: "scoped", category: "coding", source_type: :test_entry) + create_heartbeat(user: other_user, time: other_base.to_f, project: "other", category: "coding", source_type: :test_entry) + create_heartbeat(user: other_user, time: (other_base + 60.seconds).to_f, project: "other", category: "coding", source_type: :test_entry) + + spans = Clickhouse::Heartbeat.for_user(user).where(time: base.to_f...(base + 2.hours).to_f).to_span + + assert_equal 1, spans.length + assert_equal base.to_f, spans.first[:start_time] + assert_equal 60, spans.first[:duration] end end + test "generate_fields_hash is stable and insensitive to key types" do + attrs = { user_id: 7, entity: "a.rb", time: 1_700_000_000.5, language: "Ruby", is_write: true } + hash_from_symbols = Clickhouse::Heartbeat.generate_fields_hash(attrs) + hash_from_strings = Clickhouse::Heartbeat.generate_fields_hash(attrs.transform_keys(&:to_s)) + + assert_equal hash_from_symbols, hash_from_strings + assert_equal 32, hash_from_symbols.length + + different = Clickhouse::Heartbeat.generate_fields_hash(attrs.merge(entity: "b.rb")) + assert_not_equal hash_from_symbols, different + end + + test "generate_fields_hash matches the legacy Postgres digest" do + attrs = { + branch: "main", + category: "coding", + cursorpos: 17, + dependencies: [ "rails", "svelte" ], + editor: "vscode", + entity: "app/models/user.rb", + is_write: true, + language: "Ruby", + line_additions: 3, + line_deletions: 1, + lineno: 42, + lines: 120, + machine: "laptop", + operating_system: "macOS", + project: "hackatime", + project_root_count: 1, + time: 1_700_000_000.5, + type: "file", + user_agent: "vscode/1.0.0", + user_id: 123 + } + + assert_equal "2f720968efb2caf0c1f10601b724b0ca", Clickhouse::Heartbeat.generate_fields_hash(attrs) + end + + test "writer ids are JS-safe and time-ordered" do + id_now = Clickhouse::HeartbeatWriter.generate_id + id_future = Clickhouse::HeartbeatWriter.generate_id(1.hour.from_now) + + assert id_now < 2**53, "ids must stay within JS-safe integer range" + assert id_future > id_now + end + + test "merge_user_heartbeats skips rows whose latest version is deleted" do + older = User.create!(timezone: "UTC") + newer = User.create!(timezone: "UTC") + live = create_heartbeat(user: newer, time: Time.current.to_f, project: "live", source_type: :test_entry) + deleted = create_heartbeat(user: newer, time: 1.minute.from_now.to_f, project: "deleted", source_type: :test_entry) + + soft_delete_heartbeat(deleted) + Clickhouse::HeartbeatWriter.merge_user_heartbeats!(older_user_id: older.id, newer_user_id: newer.id) + + assert_equal [ live.fields_hash ], Clickhouse::Heartbeat.for_user(older).pluck(:fields_hash) + assert_empty Clickhouse::Heartbeat.for_user(newer) + end + private def create_heartbeat_sequence(user:, started_at:, editor:, count: 9) count.times do |offset| - user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/#{editor}.rb", type: "file", category: "coding", diff --git a/test/models/user_test.rb b/test/models/user_test.rb index 1c4d888f5..d14b6dbcb 100644 --- a/test/models/user_test.rb +++ b/test/models/user_test.rb @@ -250,7 +250,7 @@ class UserTest < ActiveSupport::TestCase assert_not peer.set_leaderboard_shadowban(banned: true, changed_by_user: actor, reason: "peer") end - test "changing timezone invalidates activity graph caches and schedules a dashboard rollup refresh" do + test "changing timezone invalidates activity graph caches" do with_memory_cache_store do Rails.cache.clear @@ -258,9 +258,7 @@ class UserTest < ActiveSupport::TestCase Rails.cache.write(user.activity_graph_cache_key("UTC"), { "2026-04-14" => 60 }) Rails.cache.write(user.activity_graph_cache_key("America/New_York"), { "2026-04-14" => 60 }) - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - user.update!(timezone: "America/New_York") - end + user.update!(timezone: "America/New_York") assert_not Rails.cache.exist?(user.activity_graph_cache_key("UTC")) assert_not Rails.cache.exist?(user.activity_graph_cache_key("America/New_York")) diff --git a/test/services/anonymize_user_service_test.rb b/test/services/anonymize_user_service_test.rb index 8c72c8a2a..bd61528c2 100644 --- a/test/services/anonymize_user_service_test.rb +++ b/test/services/anonymize_user_service_test.rb @@ -54,7 +54,8 @@ class AnonymizeUserServiceTest < ActiveSupport::TestCase test "anonymization soft deletes active heartbeats" do user = User.create!(username: "hb_cleanup_#{SecureRandom.hex(4)}") - heartbeat = user.heartbeats.create!( + create_heartbeat( + user: user, entity: "src/app.rb", type: "file", category: "coding", @@ -63,9 +64,13 @@ class AnonymizeUserServiceTest < ActiveSupport::TestCase source_type: :test_entry ) + total_before = Clickhouse::Heartbeat.with_deleted.where(user_id: user.id).count + assert_equal 1, Clickhouse::Heartbeat.for_user(user).count + AnonymizeUserService.call(user) - assert heartbeat.reload.deleted_at.present? + assert_equal 0, Clickhouse::Heartbeat.for_user(user).count + assert_equal total_before, Clickhouse::Heartbeat.with_deleted.where(user_id: user.id).count end test "anonymization removes legacy encrypted import credentials" do diff --git a/test/services/dashboard_rollup_refresh_service_test.rb b/test/services/dashboard_rollup_refresh_service_test.rb deleted file mode 100644 index 25d5aca0b..000000000 --- a/test/services/dashboard_rollup_refresh_service_test.rb +++ /dev/null @@ -1,72 +0,0 @@ -require "test_helper" - -class DashboardRollupRefreshServiceTest < ActiveSupport::TestCase - test "rebuilds dashboard rollups from current heartbeat aggregates" do - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - - create_heartbeat(user, "2026-04-07 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, "2026-04-07 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, "2026-04-13 10:00:00 UTC", project: nil, language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, "2026-04-13 10:01:00 UTC", project: nil, language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, "2026-04-13 10:03:00 UTC", project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - create_heartbeat(user, "2026-04-13 10:05:00 UTC", project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "browsing") - create_heartbeat(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - - total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) - filter_options_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION) - activity_graph_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - today_stats_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TODAY_STATS_DIMENSION) - - assert_equal user.heartbeats.duration_seconds, total_row.total_seconds - assert_equal user.heartbeats.count, total_row.source_heartbeats_count - assert_equal user.heartbeats.maximum(:time), total_row.source_max_heartbeat_time - - assert_equal( - user.heartbeats.group(:project).duration_seconds, - DashboardRollup.where(user: user, dimension: "project").to_h { |row| [ row.bucket, row.total_seconds ] } - ) - - language_buckets = DashboardRollup.where(user: user, dimension: "language") - .to_h { |row| [ row.bucket, row.total_seconds ] } - assert_equal Heartbeat.attributed_durations_by(user.heartbeats, :language), language_buckets - assert_equal user.heartbeats.duration_seconds, language_buckets.values.sum - assert_not_includes language_buckets.keys, "Unknown" - - assert_equal [ "alpha", "beta" ], filter_options_row.payload["project"] - assert_equal [ "javascript", "ruby" ], filter_options_row.payload["language"] - assert_equal [ "vscode", "zed" ], filter_options_row.payload["editor"] - assert_equal [ "linux", "macos" ], filter_options_row.payload["operating_system"] - assert_equal [ "browsing", "coding" ], filter_options_row.payload["category"] - - assert_equal "UTC", activity_graph_row.payload["timezone"] - assert_equal "2025-04-14", activity_graph_row.payload["start_date"] - assert_equal "2026-04-14", activity_graph_row.payload["end_date"] - assert_equal 60, activity_graph_row.payload.fetch("duration_by_date").fetch("2026-04-14") - - assert_equal "UTC", today_stats_row.payload["timezone"] - assert_equal "2026-04-14", today_stats_row.payload["today_date"] - assert_equal 60, today_stats_row.payload["todays_duration_seconds"] - assert_equal [ "Ruby" ], today_stats_row.payload["todays_language_categories"] - assert_equal [ "vscode" ], today_stats_row.payload["todays_editor_keys"] - end - end - - private - - def create_heartbeat(user, timestamp, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( - user: user, - time: Time.parse(timestamp).to_f, - project: project, - language: language, - editor: editor, - operating_system: operating_system, - category: category, - source_type: :test_entry - ) - end -end diff --git a/test/services/dashboard_stats_test.rb b/test/services/dashboard_stats_test.rb index 056923b7d..4038afd36 100644 --- a/test/services/dashboard_stats_test.rb +++ b/test/services/dashboard_stats_test.rb @@ -1,19 +1,6 @@ require "test_helper" class DashboardStatsTest < ActiveSupport::TestCase - include ActiveJob::TestHelper - - setup do - clear_enqueued_jobs - @original_queue_adapter = ActiveJob::Base.queue_adapter - ActiveJob::Base.queue_adapter = :test - end - - teardown do - clear_enqueued_jobs - ActiveJob::Base.queue_adapter = @original_queue_adapter - end - test "raw filter options are cached per user" do with_memory_cache_store do Rails.cache.clear @@ -39,25 +26,25 @@ class DashboardStatsTest < ActiveSupport::TestCase user = User.create!(timezone: "UTC") stats = build_stats(user) - Heartbeat.create!( - user: user, time: Time.current.to_f - 60, project: nil, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: Time.current.to_f - 60, project: nil, language: "ruby", editor: "vscode", operating_system: "macos", category: "coding", source_type: :test_entry ) - Heartbeat.create!( - user: user, time: Time.current.to_f, project: nil, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: Time.current.to_f, project: nil, language: "ruby", editor: "vscode", operating_system: "macos", category: "coding", source_type: :test_entry ) create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - scope = user.heartbeats + scope = Clickhouse::Heartbeat.for_user(user) assert_equal scope.group(:project).duration_seconds, stats.project_grouped_durations(scope) end - test "all-time dashboard data can be served from rollups" do + test "all-time dashboard data aggregates live heartbeats" do with_memory_cache_store do Rails.cache.clear user = User.create!(timezone: "UTC") @@ -68,143 +55,18 @@ class DashboardStatsTest < ActiveSupport::TestCase create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") travel 1.minute create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - def stats.live_raw_filter_options = raise("expected rollup-backed filter options path") - - result = stats.filterable_dashboard_data - - assert_equal user.heartbeats.duration_seconds, result[:total_time] - assert_equal user.heartbeats.count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha", "beta" ], result[:project] - end - end - - test "all-time dashboard data falls back when rollup table is unavailable" do - with_memory_cache_store do - Rails.cache.clear - user = User.create!(timezone: "UTC") - stats = build_stats(user) - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - def stats.rollups_available? = false - - result = stats.filterable_dashboard_data - - assert_equal user.heartbeats.duration_seconds, result[:total_time] - assert_equal "alpha", result["top_project"] - end - end - - test "homepage rollup path falls back to live filter options when filter option rollup is missing" do - with_memory_cache_store do - Rails.cache.clear - user = User.create!(timezone: "UTC") - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - DashboardRollupRefreshService.new(user: user).call - DashboardRollup.find_by!(user: user, dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION).destroy! - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do result = build_stats(user).filterable_dashboard_data - end - - assert_equal [ "alpha" ], result[:project] - assert_equal [ "Ruby" ], result[:language] - end - end - - test "dirty rollup serves last rollup and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - user = User.create!(timezone: "UTC") - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) - - clear_enqueued_jobs - travel 1.minute do - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) - assert_no_enqueued_jobs(only: DashboardRollupRefreshJob) do - result = stats.filterable_dashboard_data + assert_equal Clickhouse::Heartbeat.for_user(user).duration_seconds, result[:total_time] + assert_equal Clickhouse::Heartbeat.for_user(user).count, result[:total_heartbeats] + assert_equal "alpha", result["top_project"] + assert_equal [ "alpha", "beta" ], result[:project] end - - assert_equal total_row.total_seconds, result[:total_time] - assert_equal total_row.source_heartbeats_count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha" ], result[:project] end end - test "stale rollup fingerprint serves last rollup and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - user = User.create!(timezone: "UTC") - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) - - travel 1.minute do - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - DashboardRollup.clear_dirty(user.id) - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - result = stats.filterable_dashboard_data - end - - assert_equal total_row.total_seconds, result[:total_time] - assert_equal total_row.source_heartbeats_count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha" ], result[:project] - end - end - - test "today stats and activity graph can be served from rollups" do + test "today stats and activity graph are served live" do with_memory_cache_store do Rails.cache.clear @@ -214,12 +76,7 @@ def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed das create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - DashboardRollupRefreshService.new(user: user).call - stats = build_stats(user) - def stats.live_today_stats_data = raise("expected rollup-backed today stats path") - def stats.live_activity_graph_data = raise("expected rollup-backed activity graph path") - today_stats = stats.today_stats_data activity_graph = stats.activity_graph_data @@ -234,79 +91,22 @@ def stats.live_activity_graph_data = raise("expected rollup-backed activity grap end end - test "invalid today stats rollup recalculates only today stats and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - - today_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TODAY_STATS_DIMENSION) - today_row.update!(payload: today_row.payload.merge("today_date" => "2026-04-13")) - - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - def stats.live_today_stats_data = { source: :live_today } - def stats.live_activity_graph_data = raise("expected rollup-backed activity graph path") - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = today_stats = activity_graph = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = stats.filterable_dashboard_data - today_stats = stats.today_stats_data - activity_graph = stats.activity_graph_data - end - - assert_equal 120, aggregate[:total_time] - assert_equal({ source: :live_today }, today_stats) - assert_equal 120, activity_graph[:duration_by_date]["2026-04-14"] - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end - end - end - - test "invalid activity graph rollup recalculates only activity graph and schedules a refresh" do + test "unfiltered all-time dashboard data is cached" do with_memory_cache_store do Rails.cache.clear - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + user = User.create!(timezone: "UTC") + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - DashboardRollupRefreshService.new(user: user).call + first = build_stats(user).filterable_dashboard_data - activity_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - activity_row.update!(payload: activity_row.payload.merge("end_date" => "2026-04-13")) + create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - def stats.live_today_stats_data = raise("expected rollup-backed today stats path") - def stats.live_activity_graph_data = { source: :live_activity } - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = today_stats = activity_graph = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = stats.filterable_dashboard_data - today_stats = stats.today_stats_data - activity_graph = stats.activity_graph_data - end + second = build_stats(user).filterable_dashboard_data - assert_equal 120, aggregate[:total_time] - assert today_stats[:show_logged_time_sentence] - assert_equal({ source: :live_activity }, activity_graph) - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end + assert_equal first[:total_heartbeats], second[:total_heartbeats] + assert_equal first[:project], second[:project] end end @@ -320,7 +120,6 @@ def stats.live_activity_graph_data = { source: :live_activity } create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") stats = build_stats(user, params: { operating_system: "macOS" }) - def stats.rollups_available? = false result = stats.filterable_dashboard_data @@ -348,7 +147,6 @@ def stats.rollups_available? = false end end - DashboardRollupRefreshService.new(user: user).call result = build_stats(user).filterable_dashboard_data assert_equal "Linux", result["operating_system_stats"].keys.first @@ -370,15 +168,14 @@ def stats.rollups_available? = false create_heartbeat_at(user, "2026-04-13 14:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat_at(user, "2026-04-13 14:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - DashboardRollupRefreshService.new(user: user).call - result = build_stats(user).filterable_dashboard_data + result = build_stats(user).filterable_dashboard_data - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha" ], result[:project] - assert_equal({ "alpha" => 60 }, result[:project_durations]) - assert_equal({ "alpha" => 60 }, result[:weekly_project_stats].fetch("2026-04-13")) + assert_equal "alpha", result["top_project"] + assert_equal [ "alpha" ], result[:project] + assert_equal({ "alpha" => 60 }, result[:project_durations]) + assert_equal({ "alpha" => 60 }, result[:weekly_project_stats].fetch("2026-04-13")) + end end end @@ -392,7 +189,6 @@ def stats.rollups_available? = false create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") stats = build_stats(user, params: { editor: "VSCode" }) - def stats.rollups_available? = false result = stats.filterable_dashboard_data @@ -401,44 +197,6 @@ def stats.rollups_available? = false end end - test "missing today stats and activity graph rollups recalculate only those fragments and schedule one refresh" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - DashboardRollup.where( - user: user, - dimension: [ DashboardRollup::TODAY_STATS_DIMENSION, DashboardRollup::ACTIVITY_GRAPH_DIMENSION ] - ).delete_all - - stats = build_stats(user) - def stats.grouped_durations_snapshot(_scope) = raise("expected rollup-backed dashboard path") - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = today_stats = activity_graph = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = stats.filterable_dashboard_data - today_stats = stats.today_stats_data - activity_graph = stats.activity_graph_data - end - - assert_equal 120, aggregate[:total_time] - assert today_stats[:show_logged_time_sentence] - assert_equal [ ApplicationController.helpers.display_language_name("ruby") ], today_stats[:todays_languages] - assert_equal 120, activity_graph[:duration_by_date]["2026-04-14"] - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end - end - end - private def build_stats(user, params: {}) @@ -454,16 +212,16 @@ def with_memory_cache_store end def create_heartbeat(user, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( - user: user, time: Time.current.to_f, project: project, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: Time.current.to_f, project: project, language: language, editor: editor, operating_system: operating_system, category: category, source_type: :test_entry ) end def create_heartbeat_at(user, timestamp, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( - user: user, time: Time.parse(timestamp).to_f, project: project, + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, time: Time.parse(timestamp).to_f, project: project, language: language, editor: editor, operating_system: operating_system, category: category, source_type: :test_entry ) diff --git a/test/services/heartbeat_import_dump_client_test.rb b/test/services/heartbeat_import_dump_client_test.rb index 4aec68d75..2fc1aaeff 100644 --- a/test/services/heartbeat_import_dump_client_test.rb +++ b/test/services/heartbeat_import_dump_client_test.rb @@ -1,6 +1,8 @@ require "test_helper" require "webmock/minitest" +WebMock.disable_net_connect!(allow_localhost: true, allow: [ ENV.fetch("CLICKHOUSE_HOST", "clickhouse") ]) + class HeartbeatImportDumpClientTest < ActiveSupport::TestCase test "valid_wakatime_download_url? only accepts wakatime s3 links over https" do assert HeartbeatImportDumpClient.valid_wakatime_download_url?("https://wakatime.s3.amazonaws.com/export.json?signature=abc") diff --git a/test/services/heartbeat_import_service_test.rb b/test/services/heartbeat_import_service_test.rb index 40ea080b2..a2cd7fd30 100644 --- a/test/services/heartbeat_import_service_test.rb +++ b/test/services/heartbeat_import_service_test.rb @@ -30,7 +30,39 @@ class HeartbeatImportServiceTest < ActiveSupport::TestCase assert_equal 2, result[:total_count] assert_equal 1, result[:imported_count] assert_equal 1, result[:skipped_count] - assert_equal 1, user.heartbeats.count + assert_equal 1, Clickhouse::Heartbeat.for_user(user).count + end + + test "pre-batch dedupe keeps the latest heartbeat for a duplicate fields hash" do + user = User.create!(timezone: "UTC") + captured_heartbeats = nil + file_content = { + heartbeats: [ + { entity: "/tmp/newer.rb", time: 1_700_000_200.0 }, + { entity: "/tmp/older.rb", time: 1_700_000_100.0 } + ] + }.to_json + + original_normalizer = HeartbeatIngest.method(:normalize_imported_heartbeat) + original_call = HeartbeatIngest.method(:call) + HeartbeatIngest.define_singleton_method(:normalize_imported_heartbeat) do |user:, heartbeat:, user_agents_by_id: {}| + { fields_hash: "duplicate-hash", time: heartbeat.fetch("time") } + end + HeartbeatIngest.define_singleton_method(:call) do |user:, mode:, heartbeats:, user_agents_by_id: {}| + captured_heartbeats = heartbeats + HeartbeatIngest::Result.new(total_count: heartbeats.length, persisted_count: heartbeats.length, + duplicate_count: 0, failed_count: 0, errors: [], items: []) + end + + result = HeartbeatImportService.import_from_file(file_content, user) + + assert result[:success] + assert_equal 2, result[:total_count] + assert_equal 1, result[:imported_count] + assert_equal [ 1_700_000_200.0 ], captured_heartbeats.map { |heartbeat| heartbeat.fetch("time") } + ensure + HeartbeatIngest.define_singleton_method(:normalize_imported_heartbeat, original_normalizer) if original_normalizer + HeartbeatIngest.define_singleton_method(:call, original_call) if original_call end test "imports heartbeats from wakatime data dump day groups" do @@ -65,7 +97,7 @@ class HeartbeatImportServiceTest < ActiveSupport::TestCase assert_equal 1, result[:total_count] assert_equal 1, result[:imported_count] - heartbeat = user.heartbeats.order(:created_at).last + heartbeat = Clickhouse::Heartbeat.for_user(user).order(:created_at).last assert_equal "skyfall-pc", heartbeat.machine assert_equal "wakatime/v1.102.1", heartbeat.user_agent end diff --git a/test/services/heartbeat_ingest_test.rb b/test/services/heartbeat_ingest_test.rb index 98bafe9c7..4c96a94d8 100644 --- a/test/services/heartbeat_ingest_test.rb +++ b/test/services/heartbeat_ingest_test.rb @@ -16,53 +16,54 @@ class HeartbeatIngestTest < ActiveSupport::TestCase ActiveJob::Base.queue_adapter = @original_queue_adapter end - test "direct heartbeat ingest persists normalized heartbeats and schedules dashboard rollup refresh" do + def ch_count(user) = Clickhouse::Heartbeat.for_user(user).count + + test "direct heartbeat ingest persists normalized heartbeats to ClickHouse" do user = User.create!(timezone: "UTC") - assert_difference("user.heartbeats.count", 1) do - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - assert_enqueued_with(job: AttemptProjectRepoMappingJob, args: [ user.id, "hackatime" ]) do - result = HeartbeatIngest.call( - user: user, - mode: :direct, - heartbeats: [ { - entity: "src/main.rb", - plugin: "vscode/1.0.0", - project: "hackatime", - time: Time.current.to_f, - type: "file" - } ], - request_context: { - ip_address: "203.0.113.10", - machine: "laptop", - ja4: "t13d1516h2_8daaf6152771_02713d6af862" - } - ) - - assert_equal 1, result.total_count - assert_equal 1, result.persisted_count - assert_equal 0, result.duplicate_count - assert_equal 0, result.failed_count - assert_equal 1, result.items.length - assert_equal :accepted, result.items.first.status - end - end + assert_enqueued_with(job: AttemptProjectRepoMappingJob, args: [ user.id, "hackatime" ]) do + result = HeartbeatIngest.call( + user: user, + mode: :direct, + heartbeats: [ { + entity: "src/main.rb", + plugin: "vscode/1.0.0", + project: "hackatime", + time: Time.current.to_f, + type: "file" + } ], + request_context: { + ip_address: "203.0.113.10", + machine: "laptop", + ja4: "t13d1516h2_8daaf6152771_02713d6af862" + } + ) + + assert_equal 1, result.total_count + assert_equal 1, result.persisted_count + assert_equal 0, result.duplicate_count + assert_equal 0, result.failed_count + assert_equal 1, result.items.length + assert_equal :accepted, result.items.first.status end - heartbeat = user.heartbeats.order(:id).last + assert_equal 1, ch_count(user) + heartbeat = Clickhouse::Heartbeat.for_user(user).first assert_equal "vscode/1.0.0", heartbeat.user_agent assert_equal "coding", heartbeat.category assert_equal "laptop", heartbeat.machine - assert_equal "203.0.113.10", heartbeat.ip_address.to_s - assert_equal "t13d1516h2_8daaf6152771_02713d6af862", heartbeat.ja4.fingerprint + assert_equal "203.0.113.10", heartbeat.ip_address + assert_equal Ja4.find_by!(fingerprint: "t13d1516h2_8daaf6152771_02713d6af862").id, heartbeat.ja4_id assert_equal "direct_entry", heartbeat.source_type + assert heartbeat.id.positive? + assert heartbeat.fields_hash.present? end test "direct heartbeat ingest reuses a JA4 record across requests" do user = User.create!(timezone: "UTC") ja4 = "t13d1516h2_8daaf6152771_02713d6af862" - assert_difference({ "user.heartbeats.count" => 2, "Ja4.count" => 1 }) do + assert_difference("Ja4.count", 1) do HeartbeatIngest.call( user: user, mode: :direct, @@ -77,10 +78,12 @@ class HeartbeatIngestTest < ActiveSupport::TestCase ) end - assert_equal [ ja4 ], user.heartbeats.joins(:ja4).distinct.pluck("ja4s.fingerprint") + assert_equal 2, ch_count(user) + assert_equal [ Ja4.find_by!(fingerprint: ja4).id ], + Clickhouse::Heartbeat.for_user(user).distinct.pluck(:ja4_id) end - test "direct heartbeat ingest returns existing heartbeat for duplicate input" do + test "cross-request duplicate input collapses to one row under FINAL" do user = User.create!(timezone: "UTC") payload = { entity: "src/main.rb", @@ -98,24 +101,37 @@ class HeartbeatIngestTest < ActiveSupport::TestCase ) first_heartbeat = first_result.items.first.heartbeat - clear_enqueued_jobs + result = HeartbeatIngest.call( + user: user, + mode: :direct, + heartbeats: [ payload ], + request_context: { ip_address: "203.0.113.10" } + ) - assert_no_difference("user.heartbeats.count") do - result = HeartbeatIngest.call( - user: user, - mode: :direct, - heartbeats: [ payload ], - request_context: { ip_address: "203.0.113.20" } - ) + # Both requests insert; the identical (user_id, time, fields_hash) key + # dedups in ClickHouse rather than at write time. + assert_equal 1, result.persisted_count + assert_equal first_heartbeat["fields_hash"], result.items.first.heartbeat["fields_hash"] + assert_equal 1, ch_count(user) + end - assert_equal 1, result.total_count - assert_equal 0, result.persisted_count - assert_equal 1, result.duplicate_count - assert_equal 0, result.failed_count - assert_equal first_heartbeat.id, result.items.first.heartbeat.id - end + test "in-batch duplicate input is deduplicated and counted" do + user = User.create!(timezone: "UTC") + payload = { + entity: "src/main.rb", + plugin: "vscode/1.0.0", + project: "hackatime", + time: Time.current.to_f, + type: "file" + } + + result = HeartbeatIngest.call(user: user, mode: :direct, heartbeats: [ payload, payload.dup ]) - assert_no_enqueued_jobs only: DashboardRollupRefreshJob + assert_equal 2, result.total_count + assert_equal 1, result.persisted_count + assert_equal 1, result.duplicate_count + assert_equal 2, result.items.count { |item| item.status == :accepted } + assert_equal 1, ch_count(user) end test "direct heartbeat ingest resolves last language within the batch" do @@ -146,8 +162,27 @@ class HeartbeatIngestTest < ActiveSupport::TestCase ) assert_equal 2, result.persisted_count - heartbeats = user.heartbeats.order(:time) - assert_equal [ "Python", "Python" ], heartbeats.pluck(:language) + assert_equal [ "Python", "Python" ], + Clickhouse::Heartbeat.for_user(user).order(:time).pluck(:language) + end + + test "direct heartbeat ingest resolves last language from previously ingested heartbeats" do + user = User.create!(timezone: "UTC") + now = Time.current.to_f + + HeartbeatIngest.call( + user: user, + mode: :direct, + heartbeats: [ { entity: "src/first.py", time: now - 10, type: "file", language: "Python" } ] + ) + HeartbeatIngest.call( + user: user, + mode: :direct, + heartbeats: [ { entity: "src/second.xyz", time: now, type: "file", language: "<>" } ] + ) + + assert_equal [ "Python", "Python" ], + Clickhouse::Heartbeat.for_user(user).order(:time).pluck(:language) end test "direct heartbeat ingest normalizes millisecond-scaled epoch times" do @@ -160,7 +195,7 @@ class HeartbeatIngestTest < ActiveSupport::TestCase heartbeats: [ { entity: "src/main.rb", time: (sane_time * 1000).round, type: "file" } ] ) - assert_in_delta sane_time, user.heartbeats.sole.time, 1.0 + assert_in_delta sane_time, Clickhouse::Heartbeat.for_user(user).sole.time, 1.0 end test "import heartbeat ingest normalizes nanosecond-scaled epoch times" do @@ -173,7 +208,7 @@ class HeartbeatIngestTest < ActiveSupport::TestCase heartbeats: [ { entity: "/tmp/test.rb", type: "file", time: sane_time * 1_000_000_000 } ] ) - assert_in_delta sane_time, user.heartbeats.sole.time, 1.0 + assert_in_delta sane_time, Clickhouse::Heartbeat.for_user(user).sole.time, 1.0 end test "epoch normalization leaves sane and unrepairable values untouched" do @@ -186,137 +221,49 @@ class HeartbeatIngestTest < ActiveSupport::TestCase assert_equal(-9_999_999.0, ingest.send(:normalize_epoch_time, -9_999_999)) end - test "heartbeat_unique_by targets the composite index once time_epoch exists" do - ingest = HeartbeatIngest.new(user: User.new, mode: :direct, heartbeats: []) - - assert_equal [ :fields_hash ], ingest.send(:heartbeat_unique_by) - - with_time_epoch_column = Heartbeat.column_names + [ "time_epoch" ] - Heartbeat.define_singleton_method(:column_names) { with_time_epoch_column } - begin - assert_equal %i[fields_hash time_epoch], ingest.send(:heartbeat_unique_by) - ensure - Heartbeat.singleton_class.remove_method(:column_names) - end - end - - test "partition_attrs supplies floor(time) as time_epoch only once the column exists" do - ingest = HeartbeatIngest.new(user: User.new, mode: :direct, heartbeats: []) - - # pre-cutover / dev-and-test plain table: no-op - assert_equal({}, ingest.send(:partition_attrs, 1_783_000_000.9)) - - with_time_epoch_column = Heartbeat.column_names + [ "time_epoch" ] - Heartbeat.define_singleton_method(:column_names) { with_time_epoch_column } - begin - assert_equal({ time_epoch: 1_783_000_000 }, ingest.send(:partition_attrs, 1_783_000_000.9)) - assert_equal({}, ingest.send(:partition_attrs, nil)) - ensure - Heartbeat.singleton_class.remove_method(:column_names) - end - end - - test "set_time_epoch! populates the partition column when it exists" do - hb = Heartbeat.new(time: 1_783_000_000.9) - - # column absent today: hook is a no-op and does not raise - assert_nothing_raised { hb.send(:set_time_epoch!) } - - captured = [] - hb.define_singleton_method(:time_epoch=) { |v| captured << v } - with_time_epoch = Heartbeat.column_names + [ "time_epoch" ] - Heartbeat.define_singleton_method(:column_names) { with_time_epoch } - begin - hb.send(:set_time_epoch!) - assert_equal [ 1_783_000_000 ], captured - ensure - Heartbeat.singleton_class.remove_method(:column_names) - end - end - - test "with_heartbeat_unique_by re-raises inside an open transaction after refreshing the schema cache" do - ingest = HeartbeatIngest.new(user: User.new, mode: :direct, heartbeats: []) - - # transactional tests already wrap us in a transaction, so the guard applies - calls = 0 - assert_raises(ArgumentError) do - ingest.send(:with_heartbeat_unique_by) do |_unique_by| - calls += 1 - raise ArgumentError, "No unique index found" - end - end - assert_equal 1, calls - end - - test "import heartbeat ingest deduplicates imported heartbeats and schedules dashboard rollup refresh" do + test "import heartbeat ingest deduplicates imported heartbeats" do user = User.create!(timezone: "UTC") - assert_difference("user.heartbeats.count", 1) do - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - result = HeartbeatIngest.call( - user: user, - mode: :import, - heartbeats: [ - { - entity: "/tmp/test.rb", - type: "file", - time: 1_700_000_000.0, - project: "hackatime", - language: "Ruby", - is_write: true - }, - { - entity: "/tmp/test.rb", - type: "file", - time: 1_700_000_000.0, - project: "hackatime", - language: "Ruby", - is_write: true - } - ] - ) - - assert_equal 2, result.total_count - assert_equal 1, result.persisted_count - assert_equal 1, result.duplicate_count - assert_equal 0, result.failed_count - end - end + result = HeartbeatIngest.call( + user: user, + mode: :import, + heartbeats: [ + { + entity: "/tmp/test.rb", + type: "file", + time: 1_700_000_000.0, + project: "hackatime", + language: "Ruby", + is_write: true + }, + { + entity: "/tmp/test.rb", + type: "file", + time: 1_700_000_000.0, + project: "hackatime", + language: "Ruby", + is_write: true + } + ] + ) - heartbeat = user.heartbeats.order(:id).last + assert_equal 2, result.total_count + assert_equal 1, result.persisted_count + assert_equal 1, result.duplicate_count + assert_equal 0, result.failed_count + + assert_equal 1, ch_count(user) + heartbeat = Clickhouse::Heartbeat.for_user(user).sole assert_equal "wakapi_import", heartbeat.source_type end -end - -# Outside a transaction (the production configuration for both ingest modes), -# the unique_by fallback must retry exactly once with a refreshed schema cache. -class HeartbeatIngestUniqueByFallbackTest < ActiveSupport::TestCase - self.use_transactional_tests = false - test "with_heartbeat_unique_by retries once after a schema-cache failure" do - ingest = HeartbeatIngest.new(user: User.new, mode: :direct, heartbeats: []) - - calls = 0 - result = ingest.send(:with_heartbeat_unique_by) do |unique_by| - calls += 1 - raise ActiveRecord::StatementInvalid, "no unique or exclusion constraint" if calls == 1 - unique_by - end + test "re-importing the same dump does not duplicate rows" do + user = User.create!(timezone: "UTC") + heartbeats = [ { entity: "/tmp/test.rb", type: "file", time: 1_700_000_000.0, project: "hackatime", language: "Ruby", is_write: true } ] - assert_equal 2, calls - assert_equal [ :fields_hash ], result - end + HeartbeatIngest.call(user: user, mode: :import, heartbeats: heartbeats) + HeartbeatIngest.call(user: user, mode: :import, heartbeats: heartbeats) - test "with_heartbeat_unique_by does not retry more than once" do - ingest = HeartbeatIngest.new(user: User.new, mode: :direct, heartbeats: []) - - calls = 0 - assert_raises(ActiveRecord::StatementInvalid) do - ingest.send(:with_heartbeat_unique_by) do |_unique_by| - calls += 1 - raise ActiveRecord::StatementInvalid, "no unique or exclusion constraint" - end - end - assert_equal 2, calls + assert_equal 1, ch_count(user) end end diff --git a/test/services/profile_stats_service_test.rb b/test/services/profile_stats_service_test.rb index 1d0c50441..b8de6abac 100644 --- a/test/services/profile_stats_service_test.rb +++ b/test/services/profile_stats_service_test.rb @@ -5,7 +5,7 @@ class ProfileStatsServiceTest < ActiveSupport::TestCase Rails.cache.clear end - test "dashboard_stats returns dashboard-shaped data backed by rollups" do + test "dashboard_stats returns dashboard-shaped data" do user = User.create!(timezone: "UTC") base_time = (Time.current - 1.day).to_f @@ -13,8 +13,6 @@ class ProfileStatsServiceTest < ActiveSupport::TestCase create_heartbeat(user, time: base_time + 60, project: "alpha", language: "Ruby", editor: "vscode") create_heartbeat(user, time: base_time + 120, project: "beta", language: "Python", editor: "vscode") - DashboardRollupRefreshService.new(user: user).call - payload = ProfileStatsService.new(user).dashboard_stats assert payload[:filterable_dashboard_data].present? @@ -22,15 +20,13 @@ class ProfileStatsServiceTest < ActiveSupport::TestCase assert payload[:activity_graph][:duration_by_date].is_a?(Hash) end - test "og_stats reports totals and top language from rollups" do + test "og_stats reports totals and top language" do user = User.create!(timezone: "UTC") base_time = (Time.current - 1.hour).to_f create_heartbeat(user, time: base_time, project: "alpha", language: "Ruby", editor: "vscode") create_heartbeat(user, time: base_time + 60, project: "alpha", language: "Ruby", editor: "vscode") - DashboardRollupRefreshService.new(user: user).call - og = ProfileStatsService.new(user).og_stats assert_equal 60, og[:total_time_all] @@ -41,7 +37,8 @@ class ProfileStatsServiceTest < ActiveSupport::TestCase private def create_heartbeat(user, time:, project:, language: "Ruby", editor: "vscode") - user.heartbeats.create!( + Clickhouse::HeartbeatWriter.create!( + user_id: user.id, entity: "src/main.rb", type: "file", category: "coding", diff --git a/test/services/programming_goals_progress_service_test.rb b/test/services/programming_goals_progress_service_test.rb index 8f315e001..c0f450451 100644 --- a/test/services/programming_goals_progress_service_test.rb +++ b/test/services/programming_goals_progress_service_test.rb @@ -2,12 +2,12 @@ class ProgrammingGoalsProgressServiceTest < ActiveSupport::TestCase setup do - @original_timeout = Heartbeat.heartbeat_timeout_duration - Heartbeat.heartbeat_timeout_duration(1.second) + @original_timeout = Clickhouse::Heartbeat.heartbeat_timeout_duration + Clickhouse::Heartbeat.heartbeat_timeout_duration(1.second) end teardown do - Heartbeat.heartbeat_timeout_duration(@original_timeout) + Clickhouse::Heartbeat.heartbeat_timeout_duration(@original_timeout) end test "day goal uses current day in user timezone" do @@ -110,7 +110,7 @@ class ProgrammingGoalsProgressServiceTest < ActiveSupport::TestCase def create_heartbeat_pair(user, start_time, language: "Ruby", project: "alpha") start_at = to_time_in_zone(user.timezone, start_time) - Heartbeat.create!( + create_heartbeat( user: user, time: start_at.to_i, language: language, @@ -119,7 +119,7 @@ def create_heartbeat_pair(user, start_time, language: "Ruby", project: "alpha") source_type: :test_entry ) - Heartbeat.create!( + create_heartbeat( user: user, time: (start_at + 1.second).to_i, language: language, diff --git a/test/system/admin/account_merger_test.rb b/test/system/admin/account_merger_test.rb index 14bf1cc9d..d5aa27d3e 100644 --- a/test/system/admin/account_merger_test.rb +++ b/test/system/admin/account_merger_test.rb @@ -9,7 +9,7 @@ class Admin::AccountMergerTest < ApplicationSystemTestCase older.update_column(:created_at, 2.days.ago) newer.update_column(:created_at, 1.day.ago) - heartbeat = Heartbeat.create!(user: newer, time: Time.current.to_i, source_type: :test_entry) + heartbeat = create_heartbeat(user: newer, time: Time.current.to_i, source_type: :test_entry) api_key = ApiKey.create!(user: newer, name: "Merge Test Key") newer.email_addresses.create!(email: "newer@example.com", source: :signing_in) newer.sign_in_tokens.create!(auth_type: :email) @@ -50,9 +50,10 @@ class Admin::AccountMergerTest < ApplicationSystemTestCase assert_text "Merge complete!" assert_text "3 sessions/tokens revoked" - assert_text "3 related records cleaned up" + assert_text "related records cleaned up" - assert_equal older.id, Heartbeat.find(heartbeat.id).user_id + assert_equal older.id, Clickhouse::Heartbeat.for_user(older).sole.user_id + assert_equal heartbeat.id, Clickhouse::Heartbeat.for_user(older).sole.id assert_equal older.id, ApiKey.find(api_key.id).user_id assert_nil User.find_by(id: newer.id) assert_equal 0, Doorkeeper::AccessToken.where(resource_owner_id: newer.id).count diff --git a/test/system/heartbeat_export_test.rb b/test/system/heartbeat_export_test.rb index e49138c1b..e4b767510 100644 --- a/test/system/heartbeat_export_test.rb +++ b/test/system/heartbeat_export_test.rb @@ -1,7 +1,7 @@ require "application_system_test_case" class HeartbeatExportTest < ApplicationSystemTestCase - fixtures :users, :email_addresses, :heartbeats, :sign_in_tokens, :api_keys, :admin_api_keys + fixtures :users, :email_addresses, :sign_in_tokens, :api_keys, :admin_api_keys setup do @original_cache = Rails.cache @@ -115,7 +115,8 @@ def assert_latest_export_job_kwargs(expected_kwargs) end def create_heartbeat(user, at_time, entity) - user.heartbeats.create!( + super( + user: user, entity: entity, type: "file", category: "coding", diff --git a/test/system/projects_test.rb b/test/system/projects_test.rb index cb41d9136..ea44098aa 100644 --- a/test/system/projects_test.rb +++ b/test/system/projects_test.rb @@ -12,7 +12,6 @@ class ProjectsTest < ApplicationSystemTestCase archived_mapping = @user.project_repo_mappings.create!(project_name: "archived-project") archived_mapping.archive! create_project_heartbeats(@user, "archived-project", started_at: 2.days.ago.change(hour: 14)) - DashboardRollupRefreshService.new(user: @user).call visit my_projects_path @@ -62,14 +61,14 @@ class ProjectsTest < ApplicationSystemTestCase def create_project_heartbeats(user, project_name, started_at:) user.project_repo_mappings.find_or_create_by!(project_name: project_name) - Heartbeat.create!( + create_heartbeat( user: user, project: project_name, category: "coding", time: started_at.to_i, source_type: :test_entry ) - Heartbeat.create!( + create_heartbeat( user: user, project: project_name, category: "coding", diff --git a/test/test_helper.rb b/test/test_helper.rb index 75a9bc40a..4d772e5e6 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,11 +4,68 @@ require "nokogiri" require "json" +module ClickhouseTestIsolation + def before_setup + Clickhouse::Heartbeat.connection.execute("TRUNCATE TABLE heartbeats") + super + end +end + +# Heartbeats live only in ClickHouse. Tests create them through the writer and +# get back a readonly-ish Clickhouse::Heartbeat instance. +module ClickhouseHeartbeatFactory + def create_heartbeat(user: nil, user_id: nil, **attrs) + user_id ||= user.respond_to?(:id) ? user.id : user + raise ArgumentError, "create_heartbeat requires user: or user_id:" if user_id.nil? + + row = Clickhouse::HeartbeatWriter.create!(attrs.merge(user_id: user_id)) + Clickhouse::Heartbeat.instantiate(row.transform_values { |v| v.is_a?(Symbol) ? v.to_s : v }) + end + + # Soft delete = insert a tombstone version of the row. + def soft_delete_heartbeat(heartbeat) + now = Time.current + Clickhouse::HeartbeatWriter.insert_rows([ + heartbeat.attributes.slice(*Clickhouse::HeartbeatWriter::WRITABLE_COLUMNS) + .merge("deleted_at" => now, "updated_at" => now, "version" => (now.to_f * 1_000_000).round) + ]) + end + + # Restore = insert a live version with a bumped version. + def restore_heartbeat(heartbeat) + now = Time.current + Clickhouse::HeartbeatWriter.insert_rows([ + heartbeat.attributes.slice(*Clickhouse::HeartbeatWriter::WRITABLE_COLUMNS) + .merge("deleted_at" => nil, "updated_at" => now, "version" => (now.to_f * 1_000_000).round) + ]) + end +end + module ActiveSupport class TestCase # Run tests in parallel with specified workers parallelize(workers: ENV.fetch("PARALLEL_WORKERS", 2).to_i) + CLICKHOUSE_BASE_DATABASE = Clickhouse::Record.connection_db_config.database + + # Rails' TestDatabases after_fork hook has already suffixed the config's + # database with the worker number; build the name from the pre-fork base + # so this doesn't double-suffix. + parallelize_setup do |worker| + db = "#{CLICKHOUSE_BASE_DATABASE}_#{worker}" + base_config = Clickhouse::Record.connection_db_config.configuration_hash.merge(database: CLICKHOUSE_BASE_DATABASE) + Clickhouse::Record.establish_connection(base_config) + Clickhouse::Record.connection.execute("DROP DATABASE IF EXISTS #{db}") + Clickhouse::Record.connection.execute("CREATE DATABASE #{db}") + Clickhouse::Record.establish_connection(base_config.merge(database: db)) + File.read(Rails.root.join("db/clickhouse_structure.sql")).split(";\n\n").each do |statement| + Clickhouse::Record.connection.execute(statement, nil, format: nil) if statement.strip.present? + end + end + + include ClickhouseTestIsolation + include ClickhouseHeartbeatFactory + # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. fixtures :all