An error occurred while loading the file. Please try again.
-
Jayaprakash n authored2a404530
namespace: {{ flink_namespace }}imagepullsecrets: {{ imagepullsecrets }}dockerhub: {{ dockerhub }}repository: {{flink_repository|default('knowledge-platform-jobs')}}image_tag: {{ image_tag }}azure_account: {{ azure_account }}azure_secret: {{ azure_secret }}serviceMonitor: enabled: {{ service_monitor_enabled | lower}}replicaCount: {{taskmana_replicacount|default(1)}}jobmanager: rpc_port: {{ jobmanager_rpc_port }} blob_port: {{ jobmanager_blob_port }} query_port: {{ jobmanager_query_port }} ui_port: {{ jobmanager_ui_port }} prom_port: {{ jobmanager_prom_port }} heap_memory: {{ jobmanager_heap_memory }}service: {{ jobmanager_ui_service|to_json }}rest_port: {{ jobmanager_ui_rest_port }}resttcp_port: {{ jobmanager_ui_tcp_port }}taskmanager: prom_port: {{ taskmanager_prom_port }} rpc_port: {{ taskmanager_rpc_port }} heap_memory: {{ taskmanager_heap_memory }} replicas: {{taskmanager_replicacount|default(1)}}job_classname: {{ job_classname }}{{ taskmanager_liveness | to_nice_yaml }}log4j_console_properties: | # This affects logging for both user code and Flink rootLogger.level = {{ flink_jobs_console_log_level | default(INFO) }} rootLogger.appenderRef.console.ref = ConsoleAppender # Uncomment this if you want to _only_ change Flink's logging #logger.flink.name = org.apache.flink #logger.flink.level = {{ flink_jobs_console_log_level | default(INFO) }} # The following lines keep the log level of common libraries/connectors on # log level INFO. The root logger does not override this. You have to manually # change the log levels here. logger.akka.name = akka logger.akka.level = {{ flink_libraries_log_level | default(INFO) }} logger.kafka.name= org.apache.kafka logger.kafka.level = {{ flink_libraries_log_level | default(INFO) }} logger.hadoop.name = org.apache.hadoop logger.hadoop.level = {{ flink_libraries_log_level | default(INFO) }} logger.zookeeper.name = org.apache.zookeeper logger.zookeeper.level = {{ flink_libraries_log_level | default(INFO) }} # Log all infos to the console appender.console.name = ConsoleAppender appender.console.type = CONSOLE appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n # Suppress the irrelevant (wrong) warnings from the Netty channel handler logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline logger.netty.level = OFFbase_config: | kafka { broker-servers = "{{ kafka_brokers }}" zookeeper = "{{ zookeepers }}" producer {7172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
max-request-size = {{ producer_max_request_size }}
}
}
job {
env = "{{ env_name }}"
enable.distributed.checkpointing = true
statebackend {
blob {
storage {
account = "{{ azure_account }}.blob.core.windows.net"
container = "{{ flink_container_name }}"
checkpointing.dir = "checkpoint"
}
}
base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
}
}
task {
parallelism = 1
consumer.parallelism = 1
checkpointing.compressed = {{ checkpoint_compression_enabled|lower }}
checkpointing.interval = {{ checkpoint_interval }}
checkpointing.pause.between.seconds = {{ checkpoint_pause_between_seconds }}
restart-strategy.attempts = {{ restart_attempts }}
restart-strategy.delay = {{ restart_delay }} # in milli-seconds
}
redis {
host = {{ dp_redis_host }}
port = 6379
}
lms-cassandra {
host = "{{ core_cassandra_connection_ip }}"
port = "9042"
}
neo4j {
routePath = "{{ neo4j_route_path }}"
graph = "domain"
}
es {
basePath = "{{ search_es_host }}"
}
schema {
basePath = "{{ kp_schema_base_path }}"
supportedVersion = {
itemset = "2.0"
}
}
post-publish-processor:
post-publish-processor: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = {{ env_name }}.content.postpublish.request
groupId = {{ env_name }}-post-publish-processor-group
publish.topic = {{ env_name }}.publish.job.request
qrimage.topic = {{ env_name }}.qrimage.request
dialcode.context.topic = {{ env_name }}.dialcode.context.job.request
}
task {
consumer.parallelism = {{ post_publish_processor_consumer_parallelism }}
router.parallelism = {{ post_publish_event_router_parallelism }}
shallow_copy.parallelism = {{ post_publish_shallow_copy_parallelism }}
link_dialcode.parallelism = {{ post_publish_link_dialcode_parallelism }}
batch_create.parallelism = {{ post_publish_batch_create_parallelism }}
dialcode_context_updater.parallelism = {{ post_publish_dialcode_context_parallelism }}
}
lms-cassandra {
keyspace = "{{ middleware_course_keyspace }}"
batchTable = "course_batch"
}
141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
dialcode-cassandra {
keyspace = "dialcodes"
imageTable = "dialcode_images"
}
service {
search.basePath = "{{ kp_search_service_base_url }}"
lms.basePath = "{{ lms_service_base_url }}"
learning_service.basePath = "{{ kp_learning_service_base_url }}"
dial.basePath = "https://{{domain_name}}/dial/"
}
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['post-publish-processor'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
video-stream-generator:
video-stream-generator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.content.postpublish.request"
groupId = "{{ env_name }}-video-stream-generator-group"
}
task {
timer.duration = {{ video_stream_generator_timer_duration }}
consumer.parallelism = {{ video_stream_generator_consumer_parallelism }}
parallelism = {{ video_stream_generator_parallelism }}
max.retries = {{ video_stream_generator_max_retries }}
}
lms-cassandra {
keyspace = {{ platform_keyspace_name }}
table = "job_request"
}
service.content.basePath="{{ kp_content_service_base_url }}"
azure {
location = "centralindia"
login {
endpoint="https://login.microsoftonline.com"
}
api {
endpoint="https://management.azure.com"
version = "2018-07-01"
}
transform {
default = "media_transform_default"
hls = "media_transform_hls"
}
stream {
base_url="{{ video_stream_generator_azure_stream_base_url }}"
endpoint_name = "default"
protocol = "Hls"
policy_name = "Predefined_ClearStreamingOnly"
}
}
azure_tenant="{{ video_stream_generator_azure_tenant }}"
azure_subscription_id="{{ video_stream_generator_azure_subscription_id }}"
azure_account_name="{{ video_stream_generator_azure_account_name }}"
azure_resource_group_name="{{ video_stream_generator_azure_resource_group_name }}"
azure_token_client_key="{{ video_stream_generator_azure_token_client_key }}"
211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
azure_token_client_secret="{{ video_stream_generator_azure_token_client_secret }}"
## CSP Name. e.g: aws or azure
media_service_type="{{ media_service_provider_name }}"
## AWS Elemental Media Convert Config
aws {
region="{{ aws_mediaconvert_region }}"
content_bucket_name="{{ aws_content_bucket_name }}"
token {
access_key="{{ aws_mediaconvert_access_key }}"
access_secret="{{ aws_mediaconvert_access_secret }}"
}
api {
endpoint="{{ aws_mediaconvert_api_endpoint }}"
version="{{ aws_mediaconvert_api_version }}"
}
service {
name="mediaconvert"
queue="{{ aws_mediaconvert_queue_id }}"
role="{{ aws_mediaconvert_role_name }}"
}
stream {
protocol="Hls"
}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['video-stream-generator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['video-stream-generator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['video-stream-generator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
search-indexer:
search-indexer: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ search_indexer_topic_name }}"
error.topic = "{{ search_indexer_failed_topic_name }}"
groupId = "{{ search_indexer_group_name }}"
}
task {
consumer.parallelism = {{ search_indexer_consumer_parallelism }}
router.parallelism = {{ transaction_event_router_parallelism }}
compositeSearch.parallelism = {{ composite_search_indexer_parallelism }}
dialcodeIndexer.parallelism = {{ dialcode_external_indexer_parallelism }}
dialcodemetricsIndexer.parallelism = {{ dialcode_metric_indexer_parallelism }}
}
compositesearch.index.name = "{{ search_indexer_es_index_name }}"
dialcode.index.name = "{{ dialcode_es_index_name }}"
dailcodemetrics.index.name = "dialcodemetrics"
restrict.metadata.objectTypes = []
nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "transcripts", "accessibility"]
schema.definition_cache.expiry = {{ schema_definition_cache_expiry_in_sec }}
restrict.objectTypes = {{ search_indexer_restrict_object_types | to_json }}
ignored.fields={{ search_indexer_ignored_fields | to_json }}
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['search-indexer'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['search-indexer'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['search-indexer'].taskslots }}
parallelism.default: 1
281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
asset-enrichment:
asset-enrichment: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = {{ env_name }}.learning.job.request
groupId = {{ env_name }}-asset-enrichment-group
video_stream.topic = {{ env_name }}.content.postpublish.request
}
task {
checkpointing.timeout = 900000
checkpointing.interval = {{ asset_enrichment_checkpointing_interval }}
checkpointing.pause.between.seconds = {{ asset_enrichment_checkpointing_pause_interval }}
consumer.parallelism = {{ asset_enrichment_consumer_parallelism }}
router.parallelism = {{ asset_enrichment_router_parallelism }}
imageEnrichment.parallelism = {{ image_enrichment_parallelism }}
videoEnrichment.parallelism = {{ video_enrichment_parallelism }}
}
content {
stream {
enabled = true
mimeType = ["video/mp4", "video/webm"]
}
youtube {
applicationName = "fetch-youtube-license"
regexPattern = ["\\?vi?=([^&]*)", "watch\\?.*v=([^&]*)", "(?:embed|vi?)/([^/?]*)", "^([A-Za-z0-9\\-\\_]*)"]
}
upload.context.driven = true
max.iteration.count = 2
}
thumbnail.max {
sample = 5
size.pixel = 150
}
content_youtube_apikey="{{ youtube_api_key }}"
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['asset-enrichment'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['asset-enrichment'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['asset-enrichment'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
audit-history-indexer:
audit-history-indexer: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.learning.graph.events"
groupId = "{{ env_name }}-audit-history-indexer-group"
}
task {
consumer.parallelism = {{ audit_history_indexer_consumer_parallelism }}
parallelism = {{ audit_history_indexer_parallelism }}
window.time = 60
351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
}
timezone = "IST"
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['audit-history-indexer'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['audit-history-indexer'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['audit-history-indexer'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
auto-creator-v2:
auto-creator-v2: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.object.import.request"
groupId = "{{ env_name }}-auto-creator-v2-group"
}
task {
consumer.parallelism = {{ auto_creator_v2_consumer_parallelism }}
parallelism = {{ auto_creator_v2_parallelism }}
window.time = 60
}
question {
keyspace = "{{ assessment_keyspace_name }}"
}
questionset {
keyspace = "{{ hierarchy_keyspace_name }}"
}
service {
content.basePath = "{{ kp_content_service_base_url }}"
}
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
source {
baseUrl="{{ source_base_url }}"
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['auto-creator-v2'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['auto-creator-v2'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['auto-creator-v2'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
content-auto-creator:
content-auto-creator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.auto.creation.job.request"
groupId = "{{ env_name }}-content-auto-creator-group"
failed.topic = "{{ env_name }}.auto.creation.job.request.failed"
}
task {
consumer.parallelism = {{ content_auto_creator_consumer_parallelism }}
parallelism = {{ content_auto_creator_parallelism }}
window.time = 60
checkpointing.timeout = 4200000
}
redis {
421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
database {
relationCache.id = 10
collectionCache.id = 5
}
}
service {
content_service.basePath = "{{ kp_content_service_base_url }}"
search.basePath = "{{ kp_search_service_base_url }}"
lms.basePath = "{{ lms_service_base_url }}"
learning_service.basePath = "{{ kp_learning_service_base_url }}"
}
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
content_auto_creator {
actions=auto-create
allowed_object_types=["Content"]
allowed_content_stages=["create","upload","review","publish"]
content_mandatory_fields=["name","code","mimeType","primaryCategory","artifactUrl","lastPublishedBy"]
content_props_to_removed=["identifier","downloadUrl","variants","createdOn","collections","children","lastUpdatedOn","SYS_INTERNAL_LAST_UPDATED_ON","versionKey","s3Key","status","pkgVersion","toc_url","mimeTypesCount","contentTypesCount","leafNodesCount","childNodes","prevState","lastPublishedOn","flagReasons","compatibilityLevel","size","publishChecklist","publishComment","lastPublishedBy","rejectReasons","rejectComment","badgeAssertions","leafNodes","sYS_INTERNAL_LAST_UPDATED_ON","previewUrl","channel","objectType","visibility","version","pragma","prevStatus","streamingUrl","idealScreenSize","contentDisposition","lastStatusChangedOn","idealScreenDensity","lastSubmittedOn","publishError","flaggedBy","flags","lastFlaggedOn","publisher","lastUpdatedBy","lastSubmittedBy","uploadError","lockKey","publish_type","reviewError","totalCompressedSize","origin","originData","importError","questions"]
bulk_upload_mime_types=["video/mp4"]
artifact_upload_max_size=157286400
content_create_props=["name","code","mimeType","contentType","framework","processId","primaryCategory"]
artifact_upload_allowed_source=[]
g_service_acct_cred="{{ auto_creator_g_service_acct_cred }}"
gdrive.application_name=drive-download
initial_backoff_delay=120000
maximum_backoff_delay=1200000
increment_backoff_delay=2
api_call_delay=1
maxIteration=1
}
search_exists_fields=["originData"]
search_fields=["identifier","mimeType","pkgVersion","channel","status","origin","originData","artifactUrl"]
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['content-auto-creator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['content-auto-creator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['content-auto-creator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
audit-event-generator:
audit-event-generator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.learning.graph.events"
output.topic = "{{ env_name }}.telemetry.raw"
groupId = "{{ env_name }}-audit-event-generator-group"
}
task {
consumer.parallelism = {{ audit_event_generator_consumer_parallelism }}
parallelism = {{ audit_event_generator_parallelism }}
491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
producer.parallelism = {{ audit_event_generator_producer_parallelism }}
}
schema {
basePath = "{{ kp_schema_base_path }}"
}
channel.default = "{{ audit_event_generator_default_channel }}"
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['audit-event-generator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['audit-event-generator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['audit-event-generator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
metrics-data-transformer:
metrics-data-transformer: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = {{ env_name }}.learning.graph.events
groupId = {{ env_name }}-metrics-data-transformer-group
}
task {
consumer.parallelism = {{ metrics_data_transformer_consumer_parallelism }}
parallelism = {{ metrics_data_transformer_parallelism }}
producer.parallelism = {{ metrics_data_transformer_producer_parallelism }}
}
content_read_api = "/content/v3/read"
content_update_api = "/content/v4/system/update"
service {
content.basePath = "{{ kp_content_service_base_url }}"
sourcing.content.basePath = "{{ sourcing_content_service_base_url }}"
}
sourcing.update.api.response.error.code = ["404", "400"]
data.metrics = ["me_totalRatingsCount","me_averageRating","me_totalTimeSpentInSec","me_totalPlaySessionCount"]
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['metrics-data-transformer'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
mvc-indexer:
mvc-indexer: |+
include "base-config.conf"
kafka {
input.topic = "{{ env_name }}.mvc.processor.job.request"
groupId = "{{ env_name }}-mvc-indexer-group"
output.failed.topic = "{{ env_name }}.mvc.events.failed"
}
task {
consumer.parallelism = {{ mvc_indexer_consumer_parallelism }}
parallelism = {{ mvc_indexer_parallelism }}
}
lms-cassandra.keyspace = "{{ content_keyspace_name }}"
lms-cassandra.table = "content_data"
nested.fields="{{ mvc_indexer_nested_fields }}"
service.content.basePath = "{{ kp_content_service_base_url }}"
es.indexAlias = "mvc-content"
561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630
ml.vector.host="{{ mvc_indexer_ml_vector_host }}"
ml.vector.port="{{ mvc_indexer_ml_vector_port }}"
ml.keyword.host="{{ mvc_indexer_ml_keyword_host }}"
ml.keyword.port="{{ mvc_indexer_ml_keyword_port }}"
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['mvc-indexer'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['mvc-indexer'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['mvc-indexer'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
content-publish:
content-publish: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = {{ env_name }}.publish.job.request
post_publish.topic = {{ env_name }}.content.postpublish.request
mvc.topic = "{{ env_name }}.mvc.processor.job.request"
error.topic = "{{ env_name }}.learning.events.failed"
groupId = {{ env_name }}-content-publish-group
}
task {
consumer.parallelism = 1
parallelism = 1
router.parallelism = 1
}
redis {
host={{redis_host}}
port=6379
database {
contentCache.id = 0
}
}
content {
bundleLocation = "/tmp/contentBundle"
isECARExtractionEnabled = true
retry_asset_download_count = 1
keyspace = "{{ content_keyspace_name }}"
table = "content_data"
tmp_file_location = "/tmp"
objectType = ["Content", "ContentImage","Collection","CollectionImage"]
mimeType = ["application/pdf",
"application/vnd.ekstep.ecml-archive",
"application/vnd.ekstep.html-archive",
"application/vnd.android.package-archive",
"application/vnd.ekstep.content-archive",
"application/epub",
"application/msword",
"application/vnd.ekstep.h5p-archive",
"video/webm",
"video/mp4",
"application/vnd.ekstep.content-collection",
"video/quicktime",
"application/octet-stream",
"application/json",
"application/javascript",
"application/xml",
"text/plain",
"text/html",
"text/javascript",
"text/xml",
"text/css",
"image/jpeg",
"image/jpg",
"image/png",
"image/tiff",
631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700
"image/bmp",
"image/gif",
"image/svg+xml",
"image/x-quicktime",
"video/avi",
"video/mpeg",
"video/quicktime",
"video/3gpp",
"video/mp4",
"video/ogg",
"video/webm",
"video/msvideo",
"video/x-msvideo",
"video/x-qtc",
"video/x-mpeg",
"audio/mp3",
"audio/mp4",
"audio/mpeg",
"audio/ogg",
"audio/webm",
"audio/x-wav",
"audio/wav",
"audio/mpeg3",
"audio/x-mpeg-3",
"audio/vorbis",
"application/x-font-ttf",
"application/vnd.ekstep.plugin-archive",
"video/x-youtube",
"video/youtube",
"text/x-url"]
asset_download_duration = "60 seconds"
stream {
enabled = {{ content_stream_enabled | lower }}
mimeType = ["video/mp4", "video/webm"]
}
artifact.size.for_online= {{ content_artifact_size_for_online }}
downloadFiles {
spine = ["appIcon"]
full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"]
}
nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "transcripts", "accessibility"]
}
cloud_storage {
folder {
content = "content"
artifact = "artifact"
}
}
hierarchy {
keyspace = "{{ hierarchy_keyspace_name }}"
table = "content_hierarchy"
}
contentTypeToPrimaryCategory {
ClassroomTeachingVideo: "Explanation Content"
ConceptMap: "Learning Resource"
Course: "Course"
CuriosityQuestionSet: "Practice Question Set"
eTextBook: "eTextbook"
Event: "Event"
EventSet: "Event Set"
ExperientialResource: "Learning Resource"
ExplanationResource: "Explanation Content"
ExplanationVideo: "Explanation Content"
FocusSpot: "Teacher Resource"
LearningOutcomeDefinition: "Teacher Resource"
701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770
MarkingSchemeRubric: "Teacher Resource"
PedagogyFlow: "Teacher Resource"
PracticeQuestionSet: "Practice Question Set"
PracticeResource: "Practice Question Set"
SelfAssess: "Course Assessment"
TeachingMethod: "Teacher Resource"
TextBook: "Digital Textbook"
Collection: "Content Playlist"
ExplanationReadingMaterial: "Learning Resource"
LearningActivity: "Learning Resource"
LessonPlan: "Content Playlist"
LessonPlanResource: "Teacher Resource"
PreviousBoardExamPapers: "Learning Resource"
TVLesson: "Explanation Content"
OnboardingResource: "Learning Resource"
ReadingMaterial: "Learning Resource"
Template: "Template"
Asset: "Asset"
Plugin: "Plugin"
LessonPlanUnit: "Lesson Plan Unit"
CourseUnit: "Course Unit"
TextBookUnit: "Textbook Unit"
Asset: "Certificate Template"
}
compositesearch.index.name = "{{ compositesearch_index_name }}"
search.document.type = "cs"
enableDIALContextUpdate = "Yes"
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
master.category.validation.enabled ="{{ master_category_validation_enabled }}"
service {
print.basePath = "{{ kp_print_service_base_url }}"
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['content-publish'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['content-publish'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['content-publish'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
qrcode-image-generator:
qrcode-image-generator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.qrimage.request"
groupId = "{{ env_name }}-qrcode-image-generator-group"
}
task {
consumer.parallelism = {{ qrcode_image_generator_consumer_parallelism }}
parallelism = {{ qrcode_image_generator_parallelism }}
window.time = 60
}
771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840
lp.tmp.file.location="/tmp"
qr.image {
imageFormat="png"
bottomMargin=0
margin=1
}
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_dial }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_dial_bucketname | default('dial') }}"
lms-cassandra {
keyspace = "dialcodes"
table {
image = "dialcode_images"
batch = "dialcode_batch"
}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['qrcode-image-generator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['qrcode-image-generator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['qrcode-image-generator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
dialcode-context-updater:
dialcode-context-updater: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.dialcode.context.job.request"
failed.topic = "{{ env_name }}.dialcode.context.job.request.failed"
groupId = "{{ env_name }}-dialcode-group"
}
task {
consumer.parallelism = 1
parallelism = 1
dialcode-context-updater.parallelism = 1
}
dialcode_context_updater {
actions="dialcode-context-update"
search_mode="Collection"
context_map_path = "https://raw.githubusercontent.com/project-sunbird/knowledge-platform-jobs/release-5.0.0/dialcode-context-updater/src/main/resources/contextMapping.json"
identifier_search_fields = ["identifier", "primaryCategory", "channel"]
dial_code_context_read_api_path = "/dialcode/v4/read/"
dial_code_context_update_api_path = "/dialcode/v4/update/"
}
service {
search.basePath = "{{ kp_search_service_base_url }}"
dial_service.basePath = "{{ kp_dial_service_base_url }}"
}
es_sync_wait_time = 20000
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['dialcode-context-updater'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['dialcode-context-updater'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['dialcode-context-updater'].taskslots }}
parallelism.default: 1
841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
live-node-publisher:
live-node-publisher: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = {{ env_name }}.republish.job.request
live_video_stream.topic = "{{ env_name }}.live.video.stream.request"
error.topic = "{{ env_name }}.learning.events.failed"
skipped.topic = "{{ env_name }}.learning.events.skipped"
groupId = {{ env_name }}-content-republish-group
}
task {
consumer.parallelism = 1
parallelism = 1
router.parallelism = 1
}
redis {
host={{redis_host}}
port=6379
database {
contentCache.id = 0
}
}
content {
bundleLocation = "/tmp/contentBundle"
isECARExtractionEnabled = true
retry_asset_download_count = 1
keyspace = "{{ content_keyspace_name }}"
table = "content_data"
tmp_file_location = "/tmp"
objectType = ["Content", "ContentImage","Collection","CollectionImage"]
mimeType = ["application/pdf",
"application/vnd.ekstep.ecml-archive",
"application/vnd.ekstep.html-archive",
"application/vnd.android.package-archive",
"application/vnd.ekstep.content-archive",
"application/epub",
"application/msword",
"application/vnd.ekstep.h5p-archive",
"video/webm",
"video/mp4",
"application/vnd.ekstep.content-collection",
"video/quicktime",
"application/octet-stream",
"application/json",
"application/javascript",
"application/xml",
"text/plain",
"text/html",
"text/javascript",
"text/xml",
"text/css",
"image/jpeg",
"image/jpg",
"image/png",
"image/tiff",
"image/bmp",
"image/gif",
"image/svg+xml",
"image/x-quicktime",
"video/avi",
"video/mpeg",
"video/quicktime",
"video/3gpp",
"video/mp4",
"video/ogg",
"video/webm",
"video/msvideo",
911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
"video/x-msvideo",
"video/x-qtc",
"video/x-mpeg",
"audio/mp3",
"audio/mp4",
"audio/mpeg",
"audio/ogg",
"audio/webm",
"audio/x-wav",
"audio/wav",
"audio/mpeg3",
"audio/x-mpeg-3",
"audio/vorbis",
"application/x-font-ttf",
"application/vnd.ekstep.plugin-archive",
"video/x-youtube",
"video/youtube",
"text/x-url"]
asset_download_duration = "60 seconds"
stream {
enabled = {{ content_stream_enabled | lower }}
mimeType = ["video/mp4", "video/webm"]
}
artifact.size.for_online= {{ content_artifact_size_for_online }}
downloadFiles {
spine = ["appIcon"]
full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"]
}
nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "transcripts", "accessibility"]
}
cloud_storage {
folder {
content = "content"
artifact = "artifact"
}
}
hierarchy {
keyspace = "{{ hierarchy_keyspace_name }}"
table = "content_hierarchy"
}
contentTypeToPrimaryCategory {
ClassroomTeachingVideo: "Explanation Content"
ConceptMap: "Learning Resource"
Course: "Course"
CuriosityQuestionSet: "Practice Question Set"
eTextBook: "eTextbook"
Event: "Event"
EventSet: "Event Set"
ExperientialResource: "Learning Resource"
ExplanationResource: "Explanation Content"
ExplanationVideo: "Explanation Content"
FocusSpot: "Teacher Resource"
LearningOutcomeDefinition: "Teacher Resource"
MarkingSchemeRubric: "Teacher Resource"
PedagogyFlow: "Teacher Resource"
PracticeQuestionSet: "Practice Question Set"
PracticeResource: "Practice Question Set"
SelfAssess: "Course Assessment"
TeachingMethod: "Teacher Resource"
TextBook: "Digital Textbook"
Collection: "Content Playlist"
ExplanationReadingMaterial: "Learning Resource"
LearningActivity: "Learning Resource"
LessonPlan: "Content Playlist"
LessonPlanResource: "Teacher Resource"
981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050
PreviousBoardExamPapers: "Learning Resource"
TVLesson: "Explanation Content"
OnboardingResource: "Learning Resource"
ReadingMaterial: "Learning Resource"
Template: "Template"
Asset: "Asset"
Plugin: "Plugin"
LessonPlanUnit: "Lesson Plan Unit"
CourseUnit: "Course Unit"
TextBookUnit: "Textbook Unit"
Asset: "Certificate Template"
}
compositesearch.index.name = "compositesearch"
search.document.type = "cs"
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
master.category.validation.enabled ="{{ master_category_validation_enabled }}"
service {
print.basePath = "{{ kp_print_service_base_url }}"
search.basePath = "{{ kp_search_service_base_url }}"
}
cloudstorage {
metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }}
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['live-node-publisher'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['live-node-publisher'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['live-node-publisher'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
live-video-stream-generator:
live-video-stream-generator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.live.video.stream.request"
groupId = "{{ env_name }}-live-video-stream-generator-group"
}
task {
timer.duration = {{ video_stream_generator_timer_duration }}
consumer.parallelism = {{ video_stream_generator_consumer_parallelism }}
parallelism = {{ video_stream_generator_parallelism }}
max.retries = {{ video_stream_generator_max_retries }}
}
lms-cassandra {
keyspace = {{ platform_keyspace_name }}
table = "job_request"
}
service.content.basePath="{{ kp_content_service_base_url }}"
azure {
location = "centralindia"
login {
endpoint="https://login.microsoftonline.com"
}
api {
endpoint="https://management.azure.com"
version = "2018-07-01"
}
1051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120
transform {
default = "media_transform_default"
hls = "media_transform_hls"
}
stream {
base_url="{{ video_stream_generator_azure_stream_base_url }}"
endpoint_name = "default"
protocol = "Hls"
policy_name = "Predefined_ClearStreamingOnly"
}
}
azure_tenant="{{ video_stream_generator_azure_tenant }}"
azure_subscription_id="{{ video_stream_generator_azure_subscription_id }}"
azure_account_name="{{ video_stream_generator_azure_account_name }}"
azure_resource_group_name="{{ video_stream_generator_azure_resource_group_name }}"
azure_token_client_key="{{ video_stream_generator_azure_token_client_key }}"
azure_token_client_secret="{{ video_stream_generator_azure_token_client_secret }}"
## CSP Name. e.g: aws or azure
media_service_type="{{ media_service_provider_name }}"
## AWS Elemental Media Convert Config
aws {
region="{{ aws_mediaconvert_region }}"
content_bucket_name="{{ aws_content_bucket_name }}"
token {
access_key="{{ aws_mediaconvert_access_key }}"
access_secret="{{ aws_mediaconvert_access_secret }}"
}
api {
endpoint="{{ aws_mediaconvert_api_endpoint }}"
version="{{ aws_mediaconvert_api_version }}"
}
service {
name="mediaconvert"
queue="{{ aws_mediaconvert_queue_id }}"
role="{{ aws_mediaconvert_role_name }}"
}
stream {
protocol="Hls"
}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['live-video-stream-generator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['live-video-stream-generator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['live-video-stream-generator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
csp-migrator:
csp-migrator: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ csp_migration_topic_name }}"
groupId = "{{ csp_migrator_group_name }}"
failed.topic = "{{ csp_migrator_failed_topic_name }}"
live_video_stream.topic = "{{ video_stream_topic_name }}"
live_content_node_republish.topic = "{{ content_republish_topic_name }}"
live_question_node_republish.topic = "{{ question_republish_topic_name }}"
}
task {
timer.duration = {{ csp_migrator_timer_duration }}
consumer.parallelism = {{ csp_migrator_consumer_parallelism }}
parallelism = {{ csp_migrator_parallelism }}
max.retries = {{ csp_migrator_max_retries }}
cassandra-migrator.parallelism = {{csp_migrator_cassandra_parallelism}}
}
redis {
database {
1121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190
relationCache.id = 10
collectionCache.id = 5
}
}
hierarchy {
keyspace = "{{ content_hierarchy_keyspace_name }}"
table = "content_hierarchy"
}
content {
keyspace = "{{ content_keyspace_name }}"
content_table = "content_data"
assessment_table = "question_data"
}
questionset.hierarchy.keyspace="{{ questionset_hierarchy_keyspace_name }}"
questionset.hierarchy.table="questionset_hierarchy"
key_value_strings_to_migrate = {
"https://qa.ekstep.in/assets/public": "{{ cloudstorage_relative_path_prefix_content }}",
"https://dev.ekstep.in/assets/public": "{{ cloudstorage_relative_path_prefix_content }}",
"https://community.ekstep.in/assets/public": "{{ cloudstorage_relative_path_prefix_content }}",
"https://community.ekstep.in:443": "{{ cloudstorage_relative_path_prefix_content }}",
"https://ekstep-public-qa.s3-ap-south-1.amazonaws.com": "{{ cloudstorage_relative_path_prefix_content }}",
"https://ekstep-public-dev.s3-ap-south-1.amazonaws.com": "{{ cloudstorage_relative_path_prefix_content }}",
"https://ekstep-public-preprod.s3-ap-south-1.amazonaws.com": "{{ cloudstorage_relative_path_prefix_content }}",
"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com": "{{ cloudstorage_relative_path_prefix_content }}",
"https://sunbirddev.blob.core.windows.net/sunbird-content-dev": "{{ cloudstorage_relative_path_prefix_content }}",
"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging": "{{ cloudstorage_relative_path_prefix_content }}",
"https://preprodall.blob.core.windows.net/ntp-content-preprod": "{{ cloudstorage_relative_path_prefix_content }}",
"https://ntpproductionall.blob.core.windows.net/ntp-content-production": "{{ cloudstorage_relative_path_prefix_content }}",
"CLOUD_STORAGE_BASE_PATH": "{{ cloudstorage_relative_path_prefix_content }}"
}
neo4j_fields_to_migrate = {
"asset": ["artifactUrl", "thumbnail", "downloadUrl","variants"],
"content": ["appIcon", "artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon", "sourceURL", "variants", "downloadUrl","streamingUrl"],
"contentimage": ["appIcon", "artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon", "sourceURL", "variants", "downloadUrl","streamingUrl"],
"collection": ["appIcon", "artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon", "variants", "downloadUrl"],
"collectionimage": ["appIcon", "artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon", "variants", "downloadUrl"],
"plugins": ["artifactUrl"],
"itemset": ["previewUrl", "downloadUrl"],
"assessmentitem": ["data", "question", "solutions", "editorState", "media"],
"question": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"],
"questionimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"],
"questionset": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"],
"questionsetimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"]
}
cassandra_fields_to_migrate = {
"assessmentitem": ["body", "editorState", "answer", "solutions", "instructions", "media"]
}
cloudstorage {
metadata.replace_absolute_path=false
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
migrationVersion = 1
video_stream_regeneration_enable = false
live_node_republish_enable = true
copy_missing_files_to_cloud = false
download_path = /tmp
cloud_storage_type="{{ cloud_service_provider }}"
cloud_storage_key="{{ cloud_public_storage_accountname }}"
1191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242
cloud_storage_secret="{{ cloud_public_storage_secret }}"
cloud_storage_container="{{ cloud_storage_content_bucketname }}"
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['csp-migrator'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['csp-migrator'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['csp-migrator'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1
cassandra-data-migration:
cassandra-data-migration: |+
include file("/data/flink/conf/base-config.conf")
kafka {
input.topic = "{{ env_name }}.cassandra.data.migration.request"
failed.topic = "{{ env_name }}.cassandra.data.migration.job.request.failed"
groupId = "{{ env_name }}-cassandra-data-migration-group"
}
task {
consumer.parallelism = 1
parallelism = 1
}
migrate = {
keyspace = "dialcodes"
table = "dialcode_batch"
primary_key_column = "processid"
primary_key_column_type = "UUID"
column_to_migrate = "url"
column_to_migrate_type = "string"
key_value_strings_to_migrate = {
"https://sunbirdstagingpublic.blob.core.windows.net/dial": "{{ cloudstorage_relative_path_prefix_dial }}"
}
}
cloudstorage {
metadata.replace_absolute_path=false
relative_path_prefix={{ cloudstorage_relative_path_prefix_content }}
metadata.list={{ cloudstorage_metadata_list }}
read_base_path="{{ cloudstorage_base_path }}"
write_base_path={{ valid_cloudstorage_base_urls }}
}
flink-conf: |+
jobmanager.memory.flink.size: {{ flink_job_names['cassandra-data-migration'].jobmanager_memory }}
taskmanager.memory.flink.size: {{ flink_job_names['cassandra-data-migration'].taskmanager_memory }}
taskmanager.numberOfTaskSlots: {{ flink_job_names['cassandra-data-migration'].taskslots }}
parallelism.default: 1
jobmanager.execution.failover-strategy: region
taskmanager.memory.network.fraction: 0.1