Skip to content
Snippets Groups Projects
Commit 88b0e76a authored by Sarah Yasonik's avatar Sarah Yasonik Committed by Sean McGivern
Browse files

Update metrics dashboard API to load yml from repo

Updates the EnvironmentController#metrics_dashboard endpoint
to support a "dashboard" param, which can be used to specify
the filepath of a dashboard configuration from a project
repository. Dashboard configurations are expected to be
stored in .gitlab/dashboards/.

Updates dashboard post-processing steps to exclude custom
metrics, which should only display on the system dashboard.
parent 151f9e40
No related merge requests found
Showing
with 491 additions and 112 deletions
......@@ -13,6 +13,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController
before_action only: [:metrics, :additional_metrics, :metrics_dashboard] do
push_frontend_feature_flag(:metrics_time_window)
push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint)
push_frontend_feature_flag(:environment_metrics_show_multiple_dashboards)
end
def index
......@@ -158,15 +159,28 @@ def additional_metrics
end
def metrics_dashboard
return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, @project)
return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, project)
result = Gitlab::Metrics::Dashboard::Service.new(@project, @current_user, environment: environment).get_dashboard
if Feature.enabled?(:environment_metrics_show_multiple_dashboards, project)
result = dashboard_finder.find(project, current_user, environment, params[:dashboard])
result[:all_dashboards] = project.repository.metrics_dashboard_paths
else
result = dashboard_finder.find(project, current_user, environment)
end
respond_to do |format|
if result[:status] == :success
format.json { render status: :ok, json: result }
format.json do
render status: :ok, json: result.slice(:all_dashboards, :dashboard, :status)
end
else
format.json { render status: result[:http_status], json: result }
format.json do
render(
status: result[:http_status],
json: result.slice(:all_dashboards, :message, :status)
)
end
end
end
end
......@@ -211,6 +225,10 @@ def metrics_params
params.require([:start, :end])
end
def dashboard_finder
Gitlab::Metrics::Dashboard::Finder
end
def search_environment_names
return [] unless params[:query]
......
......@@ -39,7 +39,8 @@ class Repository
changelog license_blob license_key gitignore
gitlab_ci_yml branch_names tag_names branch_count
tag_count avatar exists? root_ref has_visible_content?
issue_template_names merge_request_template_names xcode_project?).freeze
issue_template_names merge_request_template_names
metrics_dashboard_paths xcode_project?).freeze
# Methods that use cache_method but only memoize the value
MEMOIZED_CACHED_METHODS = %i(license).freeze
......@@ -57,6 +58,7 @@ class Repository
avatar: :avatar,
issue_template: :issue_template_names,
merge_request_template: :merge_request_template_names,
metrics_dashboard: :metrics_dashboard_paths,
xcode_config: :xcode_project?
}.freeze
......@@ -602,6 +604,11 @@ def merge_request_template_names
end
cache_method :merge_request_template_names, fallback: []
def metrics_dashboard_paths
Gitlab::Metrics::Dashboard::Finder.find_all_paths_from_source(project)
end
cache_method :metrics_dashboard_paths
def readme
head_tree&.readme
end
......
......@@ -12,7 +12,7 @@ module Processor
].freeze
override :sequence
def sequence
def sequence(_insert_project_metrics)
super + EE_SEQUENCE
end
end
......
......@@ -13,7 +13,7 @@ class AlertsInserter < ::Gitlab::Metrics::Dashboard::Stages::BaseStage
def transform!
return if metrics_with_alerts.empty?
for_metrics(dashboard) do |metric|
for_metrics do |metric|
next unless metrics_with_alerts.include?(metric[:metric_id])
metric[:alert_path] = alert_path(metric[:metric_id], project, environment)
......
......@@ -10,7 +10,7 @@
describe 'sequence' do
let(:environment) { build(:environment) }
let(:sequence) { described_class.new(*params).sequence }
let(:sequence) { described_class.new(*params).__send__(:sequence, insert_project_metrics: true) }
it 'includes the alerts processing stage' do
expect(sequence.length).to eq(4)
......@@ -18,7 +18,7 @@
end
describe 'process' do
let(:dashboard) { described_class.new(*params).process }
let(:dashboard) { described_class.new(*params).process(insert_project_metrics: true) }
context 'when the dashboard references persisted metrics with alerts' do
let!(:alert) do
......
......@@ -16,6 +16,7 @@ module FileDetector
avatar: /\Alogo\.(png|jpg|gif)\z/,
issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z},
merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z},
metrics_dashboard: %r{\A\.gitlab/dashboards/[^/]+\.yml\z},
xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z},
# Configuration files
......
# frozen_string_literal: true
# Searches a projects repository for a metrics dashboard and formats the output.
# Expects any custom dashboards will be located in `.gitlab/dashboards`
module Gitlab
module Metrics
module Dashboard
class BaseService < ::BaseService
DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError
def get_dashboard
return error("#{dashboard_path} could not be found.", :not_found) unless path_available?
success(dashboard: process_dashboard)
rescue DASHBOARD_LAYOUT_ERROR => e
error(e.message, :unprocessable_entity)
end
# Summary of all known dashboards for the service.
# @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
def all_dashboard_paths(_project)
raise NotImplementedError
end
private
# Returns a new dashboard Hash, supplemented with DB info
def process_dashboard
Gitlab::Metrics::Dashboard::Processor
.new(project, params[:environment], raw_dashboard)
.process(insert_project_metrics: insert_project_metrics?)
end
# @return [String] Relative filepath of the dashboard yml
def dashboard_path
params[:dashboard_path]
end
# Returns an un-processed dashboard from the cache.
def raw_dashboard
Rails.cache.fetch(cache_key) { get_raw_dashboard }
end
# @return [Hash] an unmodified dashboard
def get_raw_dashboard
raise NotImplementedError
end
# @return [String]
def cache_key
raise NotImplementedError
end
# Determines whether custom metrics should be included
# in the processed output.
def insert_project_metrics?
false
end
# Checks if dashboard path exists or should be rejected
# as a result of file-changes to the project repository.
# @return [Boolean]
def path_available?
available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project)
available_paths.any? do |path_params|
path_params[:path] == dashboard_path
end
end
end
end
end
end
# frozen_string_literal: true
# Returns DB-supplmented dashboard info for determining
# the layout of UI. Intended entry-point for the Metrics::Dashboard
# module.
module Gitlab
module Metrics
module Dashboard
class Finder
class << self
# Returns a formatted dashboard packed with DB info.
# @return [Hash]
def find(project, user, environment, dashboard_path = nil)
service = system_dashboard?(dashboard_path) ? system_service : project_service
service
.new(project, user, environment: environment, dashboard_path: dashboard_path)
.get_dashboard
end
# Summary of all known dashboards.
# @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
def find_all_paths(project)
project.repository.metrics_dashboard_paths
end
# Summary of all known dashboards. Used to populate repo cache.
# Prefer #find_all_paths.
def find_all_paths_from_source(project)
system_service.all_dashboard_paths(project)
.+ project_service.all_dashboard_paths(project)
end
private
def system_service
Gitlab::Metrics::Dashboard::SystemDashboardService
end
def project_service
Gitlab::Metrics::Dashboard::ProjectDashboardService
end
def system_dashboard?(filepath)
!filepath || system_service.system_dashboard?(filepath)
end
end
end
end
end
end
......@@ -8,12 +8,17 @@ module Dashboard
# the UI. These includes shared metric info, custom metrics
# info, and alerts (only in EE).
class Processor
SEQUENCE = [
SYSTEM_SEQUENCE = [
Stages::CommonMetricsInserter,
Stages::ProjectMetricsInserter,
Stages::Sorter
].freeze
PROJECT_SEQUENCE = [
Stages::CommonMetricsInserter,
Stages::Sorter
].freeze
def initialize(project, environment, dashboard)
@project = project
@environment = environment
......@@ -22,9 +27,9 @@ def initialize(project, environment, dashboard)
# Returns a new dashboard hash with the results of
# running transforms on the dashboard.
def process
def process(insert_project_metrics:)
@dashboard.deep_symbolize_keys.tap do |dashboard|
sequence.each do |stage|
sequence(insert_project_metrics).each do |stage|
stage.new(@project, @environment, dashboard).transform!
end
end
......@@ -32,8 +37,8 @@ def process
private
def sequence
SEQUENCE
def sequence(insert_project_metrics)
insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE
end
end
end
......
# frozen_string_literal: true
# Searches a projects repository for a metrics dashboard and formats the output.
# Expects any custom dashboards will be located in `.gitlab/dashboards`
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Gitlab
module Metrics
module Dashboard
class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService
DASHBOARD_ROOT = ".gitlab/dashboards"
class << self
def all_dashboard_paths(project)
file_finder(project)
.list_files_for(DASHBOARD_ROOT)
.map do |filepath|
Rails.cache.delete(cache_key(project.id, filepath))
{ path: filepath, default: false }
end
end
def file_finder(project)
Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml')
end
def cache_key(id, dashboard_path)
"project_#{id}_metrics_dashboard_#{dashboard_path}"
end
end
private
# Searches the project repo for a custom-defined dashboard.
def get_raw_dashboard
yml = self.class.file_finder(project).read(dashboard_path)
YAML.safe_load(yml)
end
def cache_key
self.class.cache_key(project.id, dashboard_path)
end
end
end
end
end
# frozen_string_literal: true
# Fetches the metrics dashboard layout and supplemented the output with DB info.
module Gitlab
module Metrics
module Dashboard
class Service < ::BaseService
SYSTEM_DASHBOARD_NAME = 'common_metrics'
SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml")
# Returns a DB-supplemented json representation of a dashboard config file.
def get_dashboard
dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard }
dashboard = process_dashboard(dashboard_string)
success(dashboard: dashboard)
rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e
error(e.message, :unprocessable_entity)
end
private
# Returns the base metrics shipped with every GitLab service.
def system_dashboard
YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH))
end
def cache_key
"metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}"
end
# Returns a new dashboard Hash, supplemented with DB info
def process_dashboard(dashboard)
Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process
end
end
end
end
end
......@@ -36,7 +36,7 @@ def missing_metrics!
raise DashboardLayoutError.new('Each "panel" must define an array :metrics')
end
def for_metrics(dashboard)
def for_metrics
missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array)
dashboard[:panel_groups].each do |panel_group|
......
......@@ -11,7 +11,7 @@ class CommonMetricsInserter < BaseStage
def transform!
common_metrics = ::PrometheusMetric.common
for_metrics(dashboard) do |metric|
for_metrics do |metric|
metric_record = common_metrics.find { |m| m.identifier == metric[:id] }
metric[:metric_id] = metric_record.id if metric_record
end
......
# frozen_string_literal: true
# Fetches the system metrics dashboard and formats the output.
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Gitlab
module Metrics
module Dashboard
class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService
SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml'
class << self
def all_dashboard_paths(_project)
[{
path: SYSTEM_DASHBOARD_PATH,
default: true
}]
end
def system_dashboard?(filepath)
filepath == SYSTEM_DASHBOARD_PATH
end
end
private
def dashboard_path
SYSTEM_DASHBOARD_PATH
end
# Returns the base metrics shipped with every GitLab service.
def get_raw_dashboard
yml = File.read(Rails.root.join(dashboard_path))
YAML.safe_load(yml)
end
def cache_key
"metrics_dashboard_#{dashboard_path}"
end
def insert_project_metrics?
true
end
end
end
end
end
......@@ -474,25 +474,102 @@
end
end
context 'when prometheus endpoint is enabled' do
shared_examples_for '200 response' do |contains_all_dashboards: false|
let(:expected_keys) { %w(dashboard status) }
before do
expected_keys << 'all_dashboards' if contains_all_dashboards
end
it 'returns a json representation of the environment dashboard' do
get :metrics_dashboard, params: environment_params(format: :json)
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.keys).to contain_exactly('dashboard', 'status')
expect(json_response.keys).to contain_exactly(*expected_keys)
expect(json_response['dashboard']).to be_an_instance_of(Hash)
end
end
shared_examples_for 'error response' do |status_code, contains_all_dashboards: false|
let(:expected_keys) { %w(message status) }
before do
expected_keys << 'all_dashboards' if contains_all_dashboards
end
it 'returns an error response' do
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(response).to have_gitlab_http_status(status_code)
expect(json_response.keys).to contain_exactly(*expected_keys)
end
end
shared_examples_for 'has all dashboards' do
it 'includes an index of all available dashboards' do
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(json_response.keys).to include('all_dashboards')
expect(json_response['all_dashboards']).to be_an_instance_of(Array)
expect(json_response['all_dashboards']).to all( include('path', 'default') )
end
end
context 'when multiple dashboards is disabled' do
before do
stub_feature_flags(environment_metrics_show_multiple_dashboards: false)
end
let(:dashboard_params) { { format: :json } }
it_behaves_like '200 response'
context 'when the dashboard could not be provided' do
before do
allow(YAML).to receive(:safe_load).and_return({})
end
it 'returns an error response' do
get :metrics_dashboard, params: environment_params(format: :json)
it_behaves_like 'error response', :unprocessable_entity
end
context 'when a dashboard param is specified' do
let(:dashboard_params) { { format: :json, dashboard: '.gitlab/dashboards/not_there_dashboard.yml' } }
it_behaves_like '200 response'
end
end
context 'when multiple dashboards is enabled' do
let(:dashboard_params) { { format: :json } }
it_behaves_like '200 response', contains_all_dashboards: true
it_behaves_like 'has all dashboards'
context 'when a dashboard could not be provided' do
before do
allow(YAML).to receive(:safe_load).and_return({})
end
it_behaves_like 'error response', :unprocessable_entity, contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
context 'when a dashboard param is specified' do
let(:dashboard_params) { { format: :json, dashboard: '.gitlab/dashboards/test.yml' } }
context 'when the dashboard is available' do
let(:dashboard_yml) { fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
let(:dashboard_file) { { '.gitlab/dashboards/test.yml' => dashboard_yml } }
let(:project) { create(:project, :custom_repo, files: dashboard_file) }
let(:environment) { create(:environment, name: 'production', project: project) }
it_behaves_like '200 response', contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
expect(response).to have_gitlab_http_status(:unprocessable_entity)
expect(json_response.keys).to contain_exactly('message', 'status', 'http_status')
context 'when the dashboard does not exist' do
it_behaves_like 'error response', :not_found, contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
end
end
......
......@@ -2,7 +2,7 @@ dashboard: 'Test Dashboard'
priority: 1
panel_groups:
- group: Group A
priority: 10
priority: 1
panels:
- title: "Super Chart A1"
type: "area-chart"
......@@ -23,7 +23,7 @@ panel_groups:
label: Legend Label
unit: unit
- group: Group B
priority: 1
priority: 10
panels:
- title: "Super Chart B"
type: "area-chart"
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store_caching do
include MetricsDashboardHelpers
set(:project) { build(:project) }
set(:environment) { build(:environment, project: project) }
let(:system_dashboard_path) { Gitlab::Metrics::Dashboard::SystemDashboardService::SYSTEM_DASHBOARD_PATH}
describe '.find' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:service_call) { described_class.find(project, nil, environment, dashboard_path) }
it_behaves_like 'misconfigured dashboard service response', :not_found
context 'when the dashboard exists' do
let(:project) { project_with_dashboard(dashboard_path) }
it_behaves_like 'valid dashboard service response'
end
context 'when the dashboard is configured incorrectly' do
let(:project) { project_with_dashboard(dashboard_path, {}) }
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when the system dashboard is specified' do
let(:dashboard_path) { system_dashboard_path }
it_behaves_like 'valid dashboard service response'
end
context 'when no dashboard is specified' do
let(:service_call) { described_class.find(project, nil, environment) }
it_behaves_like 'valid dashboard service response'
end
end
describe '.find_all_paths' do
let(:all_dashboard_paths) { described_class.find_all_paths(project) }
let(:system_dashboard) { { path: system_dashboard_path, default: true } }
it 'includes only the system dashboard by default' do
expect(all_dashboard_paths).to eq([system_dashboard])
end
context 'when the project contains dashboards' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:project) { project_with_dashboard(dashboard_path) }
it 'includes system and project dashboards' do
project_dashboard = { path: dashboard_path, default: false }
expect(all_dashboard_paths).to contain_exactly(system_dashboard, project_dashboard)
end
end
end
end
......@@ -4,12 +4,12 @@
describe Gitlab::Metrics::Dashboard::Processor do
let(:project) { build(:project) }
let(:environment) { build(:environment) }
let(:environment) { build(:environment, project: project) }
let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
describe 'process' do
let(:process_params) { [project, environment, dashboard_yml] }
let(:dashboard) { described_class.new(*process_params).process }
let(:dashboard) { described_class.new(*process_params).process(insert_project_metrics: true) }
context 'when dashboard config corresponds to common metrics' do
let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') }
......@@ -35,9 +35,9 @@
it 'orders groups by priority and panels by weight' do
expected_metrics_order = [
'metric_a2', # group priority 10, panel weight 2
'metric_a1', # group priority 10, panel weight 1
'metric_b', # group priority 1, panel weight 1
'metric_b', # group priority 10, panel weight 1
'metric_a2', # group priority 1, panel weight 2
'metric_a1', # group priority 1, panel weight 1
project_business_metric.id, # group priority 0, panel weight nil (0)
project_response_metric.id, # group priority -5, panel weight nil (0)
project_system_metric.id, # group priority -10, panel weight nil (0)
......@@ -46,6 +46,17 @@
expect(actual_metrics_order).to eq expected_metrics_order
end
context 'when the dashboard should not include project metrics' do
let(:dashboard) { described_class.new(*process_params).process(insert_project_metrics: false) }
it 'includes only dashboard metrics' do
metrics = all_metrics.map { |m| m[:id] }
expect(metrics.length).to be(3)
expect(metrics).to eq %w(metric_b metric_a2 metric_a1)
end
end
end
shared_examples_for 'errors with message' do |expected_message|
......
# frozen_string_literal: true
require 'rails_helper'
describe Gitlab::Metrics::Dashboard::ProjectDashboardService, :use_clean_rails_memory_store_caching do
include MetricsDashboardHelpers
set(:user) { build(:user) }
set(:project) { build(:project) }
set(:environment) { build(:environment, project: project) }
before do
project.add_maintainer(user)
end
describe 'get_dashboard' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:service_params) { [project, user, { environment: environment, dashboard_path: dashboard_path }] }
let(:service_call) { described_class.new(*service_params).get_dashboard }
context 'when the dashboard does not exist' do
it_behaves_like 'misconfigured dashboard service response', :not_found
end
context 'when the dashboard exists' do
let(:project) { project_with_dashboard(dashboard_path) }
it_behaves_like 'valid dashboard service response'
it 'caches the unprocessed dashboard for subsequent calls' do
expect_any_instance_of(described_class)
.to receive(:get_raw_dashboard)
.once
.and_call_original
described_class.new(*service_params).get_dashboard
described_class.new(*service_params).get_dashboard
end
context 'and the dashboard is then deleted' do
it 'does not return the previously cached dashboard' do
described_class.new(*service_params).get_dashboard
delete_project_dashboard(project, user, dashboard_path)
expect_any_instance_of(described_class)
.to receive(:get_raw_dashboard)
.once
.and_call_original
described_class.new(*service_params).get_dashboard
end
end
end
context 'when the dashboard is configured incorrectly' do
let(:project) { project_with_dashboard(dashboard_path, {}) }
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Service, :use_clean_rails_memory_store_caching do
let(:project) { build(:project) }
let(:environment) { build(:environment) }
describe 'get_dashboard' do
let(:dashboard_schema) { JSON.parse(fixture_file('lib/gitlab/metrics/dashboard/schemas/dashboard.json')) }
it 'returns a json representation of the environment dashboard' do
result = described_class.new(project, environment).get_dashboard
expect(result.keys).to contain_exactly(:dashboard, :status)
expect(result[:status]).to eq(:success)
expect(JSON::Validator.fully_validate(dashboard_schema, result[:dashboard])).to be_empty
end
it 'caches the dashboard for subsequent calls' do
expect(YAML).to receive(:safe_load).once.and_call_original
described_class.new(project, environment).get_dashboard
described_class.new(project, environment).get_dashboard
end
context 'when the dashboard is configured incorrectly' do
before do
allow(YAML).to receive(:safe_load).and_return({})
end
it 'returns an appropriate message and status code' do
result = described_class.new(project, environment).get_dashboard
expect(result.keys).to contain_exactly(:message, :http_status, :status)
expect(result[:status]).to eq(:error)
expect(result[:http_status]).to eq(:unprocessable_entity)
end
end
end
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment