Unverified Commit 725ce3bf authored by Keshav Prasad's avatar Keshav Prasad Committed by GitHub
Browse files

fix: add json parser for opa logs (#3174)

Showing with 1 addition and 1 deletion
+1 -1
{"_id":{"$oid":"606d4adecc575336d5ef0c5f"},"creator_user_id":"admin","configuration":{"recv_buffer_size":1048576,"tcp_keepalive":true,"number_worker_threads":16,"tls_client_auth_cert_file":"","bind_address":"0.0.0.0","tls_cert_file":"","port":5044,"tls_key_file":"","tls_enable":false,"tls_key_password":"","tls_client_auth":"disabled","override_source":null,"no_beats_prefix":true},"name":"Beats","created_at":{"$date":"2021-04-07T09:09:23.771Z"},"global":true,"type":"org.graylog.plugins.beats.Beats2Input","title":"beats","content_pack":null}
{"_id":{"$oid":"606d4adecc575336d5ef0c5f"},"creator_user_id":"admin","configuration":{"recv_buffer_size":1048576,"tcp_keepalive":true,"number_worker_threads":16,"tls_client_auth_cert_file":"","bind_address":"0.0.0.0","tls_cert_file":"","port":5044,"tls_key_file":"","tls_enable":false,"tls_key_password":"","tls_client_auth":"disabled","override_source":null,"no_beats_prefix":true},"name":"Beats","created_at":{"$date":"2021-04-07T09:09:23.771Z"},"global":true,"type":"org.graylog.plugins.beats.Beats2Input","title":"beats","content_pack":null,"extractors":[{"creator_user_id":"admin","source_field":"message","condition_type":"string","title":"JSON_EXTRACTOR_FOR_OPA_LOGS","type":"json","cursor_strategy":"copy","target_field":"","extractor_config":{"flatten":false,"list_separator":", ","kv_separator":"=","key_prefix":"opa_","key_separator":".","replace_key_whitespace":false,"key_whitespace_replacement":""},"condition_value":"decision_id","converters":[],"id":"840a58c0-84f4-11ec-b9c2-002248d48142","order":{"$numberLong":"0"}}]}
{"_id":{"$oid":"608fa8f9feca855051cffa19"},"creator_user_id":"admin","extractors":[{"creator_user_id":"admin","source_field":"message","condition_type":"none","title":"JSON_EXTRACTOR","type":"json","cursor_strategy":"cut","target_field":"","extractor_config":{"flatten":false,"list_separator":", ","kv_separator":"=","key_prefix":"","key_separator":"_","replace_key_whitespace":true,"key_whitespace_replacement":"_"},"condition_value":"","converters":[],"id":"2cf74d70-abe4-11eb-8d7f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"source","condition_type":"none","title":"SET_SOURCE","type":"regex_replace","cursor_strategy":"cut","target_field":"source","extractor_config":{"regex":"^unknown$","replacement":"druid.events.log","replace_all":true},"condition_value":"","converters":[],"id":"e728f490-abea-11eb-8d7f-002248d48142","order":{"$numberLong":"1"}}],"configuration":{"topic_filter":"^.*.druid.events.log$","fetch_wait_max":100,"offset_reset":"largest","zookeeper":"","threads":1,"custom_properties":"","group_id":"graylog2","throttling_allowed":false,"fetch_min_bytes":5,"legacy_mode":false,"override_source":null,"bootstrap_server":"{{ sunbird_processing_kafka_host }}"},"name":"Raw/Plaintext Kafka","created_at":{"$date":"2021-05-03T13:42:16.955Z"},"global":true,"type":"org.graylog2.inputs.raw.kafka.RawKafkaInput","title":"druid.events.log","content_pack":null}
{"_id":{"$oid":"608fb5affeca855051d007ee"},"creator_user_id":"admin","extractors":[{"creator_user_id":"admin","source_field":"message","condition_type":"none","title":"JSON_EXTRACTOR","type":"json","cursor_strategy":"cut","target_field":"","extractor_config":{"flatten":false,"list_separator":", ","kv_separator":"=","key_prefix":"","key_separator":"_","replace_key_whitespace":true,"key_whitespace_replacement":"_"},"condition_value":"","converters":[],"id":"7b87be50-abeb-11eb-8d7f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"source","condition_type":"none","title":"SET_SOURCE","type":"regex_replace","cursor_strategy":"cut","target_field":"source","extractor_config":{"regex":"^unknown$","replacement":"druid.events.error","replace_all":true},"condition_value":"","converters":[],"id":"9e4ddfa0-abeb-11eb-8d7f-002248d48142","order":{"$numberLong":"0"}}],"configuration":{"topic_filter":"^.*.druid.events.error$","fetch_wait_max":100,"offset_reset":"largest","zookeeper":"","threads":1,"custom_properties":"","group_id":"graylog2","throttling_allowed":false,"fetch_min_bytes":5,"legacy_mode":false,"override_source":null,"bootstrap_server":"{{ sunbird_processing_kafka_host }}"},"name":"Raw/Plaintext Kafka","created_at":{"$date":"2021-05-03T13:42:05.635Z"},"global":true,"type":"org.graylog2.inputs.raw.kafka.RawKafkaInput","title":"druid.events.error","content_pack":null}
{"_id":{"$oid":"60e58a23dc039a5b3f22ab06"},"creator_user_id":"admin","extractors":[{"creator_user_id":"admin","source_field":"message","condition_type":"none","title":"JSON EXTRACTOR","type":"json","cursor_strategy":"copy","target_field":"","extractor_config":{"flatten":false,"list_separator":", ","kv_separator":"=","key_prefix":"","key_separator":"_","replace_key_whitespace":false,"key_whitespace_replacement":"_"},"condition_value":"","converters":[],"id":"3411a150-df15-11eb-a1a4-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"source","condition_type":"none","title":"SET SOURCE","type":"regex_replace","cursor_strategy":"copy","target_field":"source","extractor_config":{"regex":"^unknown$","replacement":"dataproducts.metrics","replace_all":true},"condition_value":"","converters":[],"id":"2cdd1e70-df18-11eb-a1a4-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"dimensions","condition_type":"none","title":"DIMENSION EXTRACTOR","type":"grok","cursor_strategy":"copy","target_field":"","extractor_config":{"grok_pattern":"%{DP_DIMENSION_PARSE}"},"condition_value":"","converters":[],"id":"e1c0f630-e0a4-11eb-8a7a-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"metrics","condition_type":"none","title":"EXHAUST METRICS EXTRACTOR","type":"grok","cursor_strategy":"copy","target_field":"","extractor_config":{"grok_pattern":"%{DP_METRIC_PARSE_2}"},"condition_value":"","converters":[],"id":"45ecf160-e468-11eb-8a7a-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"metrics","condition_type":"none","title":"METRICS EXTRACTOR","type":"grok","cursor_strategy":"copy","target_field":"","extractor_config":{"grok_pattern":"%{DP_METRIC_PARSE_1}"},"condition_value":"","converters":[],"id":"05259460-e469-11eb-8a7a-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"time-taken-secs","condition_type":"none","title":"DP_Time_Taken_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"time_taken_secs","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"93a03df0-178e-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"total-requests","condition_type":"none","title":"DP_Total_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"total_requests","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"85e86720-1791-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"success-requests","condition_type":"none","title":"DP_Success_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"success_requests","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"a166fa20-1791-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"failed-requests","condition_type":"none","title":"DP_Failed_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"failed_requests","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"bb4fd1f0-1791-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"input-events","condition_type":"none","title":"DP_Input_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"input_events","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"ebca8700-1794-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"output-events","condition_type":"none","title":"DP_Output_Numeric_Convertor","type":"copy_input","cursor_strategy":"cut","target_field":"output_events","extractor_config":{},"condition_value":"","converters":[{"type":"numeric","config":{}}],"id":"049c4980-1795-11ec-bf6f-002248d48142","order":{"$numberLong":"0"}},{"creator_user_id":"admin","source_field":"metrics","condition_type":"none","title":"FILESIZE METRICS EXTRACTOR","type":"grok","cursor_strategy":"copy","target_field":"","extractor_config":{"grok_pattern":"%{DP_METRICS_PARSE_3}"},"condition_value":"","converters":[],"id":"bccd12c0-3c96-11ec-b6ec-002248d48142","order":{"$numberLong":"0"}}],"configuration":{"topic_filter":"^.*.prom.monitoring.metrics$","fetch_wait_max":100,"offset_reset":"largest","zookeeper":"","threads":2,"custom_properties":"","group_id":"graylog2","throttling_allowed":false,"fetch_min_bytes":5,"legacy_mode":false,"override_source":null,"bootstrap_server":"{{ sunbird_processing_kafka_host }}"},"name":"Raw/Plaintext Kafka","created_at":{"$date":"2021-07-15T10:36:58.024Z"},"global":true,"type":"org.graylog2.inputs.raw.kafka.RawKafkaInput","title":"DataProducts","content_pack":null,"node_id":"a02291b8-a5df-409b-8c6e-8e70d8280ffa"}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment