Merge branch 'release-3.9.0' into questionset-instructions

assessment-api/assessment-actors/pom.xml

@@ -37,6 +37,12 @@
             <artifactId>qs-hierarchy-manager</artifactId>
             <version>1.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>org.sunbird</groupId>
+            <artifactId>import-manager</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.maj.version}</artifactId>

assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionActor.scala

@@ -3,8 +3,9 @@ package org.sunbird.actors
 import java.util
 
 import javax.inject.Inject
+import org.sunbird.`object`.importer.{ImportConfig, ImportManager}
 import org.sunbird.actor.core.BaseActor
-import org.sunbird.common.DateUtils
+import org.sunbird.common.{DateUtils, Platform}
 import org.sunbird.common.dto.{Request, Response, ResponseHandler}
 import org.sunbird.graph.OntologyEngineContext
 import org.sunbird.graph.nodes.DataNode
@@ -18,6 +19,9 @@ class QuestionActor @Inject()(implicit oec: OntologyEngineContext) extends BaseA
 
 	implicit val ec: ExecutionContext = getContext().dispatcher
 
+	private lazy val importConfig = getImportConfig()
+	private lazy val importMgr = new ImportManager(importConfig)
+
 	override def onReceive(request: Request): Future[Response] = request.getOperation match {
 		case "createQuestion" => AssessmentManager.create(request, "ERR_QUESTION_CREATE")
 		case "readQuestion" => AssessmentManager.read(request, "question")
@@ -25,6 +29,7 @@ class QuestionActor @Inject()(implicit oec: OntologyEngineContext) extends BaseA
 		case "reviewQuestion" => review(request)
 		case "publishQuestion" => publish(request)
 		case "retireQuestion" => retire(request)
+		case "importQuestion" => importQuestion(request)
 		case _ => ERROR(request.getOperation)
 	}
 
@@ -69,4 +74,15 @@ class QuestionActor @Inject()(implicit oec: OntologyEngineContext) extends BaseA
 		})
 	}
 
+	def importQuestion(request: Request): Future[Response] = importMgr.importObject(request)
+
+	def getImportConfig(): ImportConfig = {
+		val requiredProps = Platform.getStringList("import.required_props.question", java.util.Arrays.asList("name", "code", "mimeType", "framework")).asScala.toList
+		val validStages = Platform.getStringList("import.valid_stages.question", java.util.Arrays.asList("create", "upload", "review", "publish")).asScala.toList
+		val propsToRemove = Platform.getStringList("import.remove_props.question", java.util.Arrays.asList()).asScala.toList
+		val topicName = Platform.config.getString("import.output_topic_name")
+		val reqLimit = Platform.getInteger("import.request_size_limit", 200)
+		ImportConfig(topicName, reqLimit, requiredProps, validStages, propsToRemove)
+	}
+
 }

assessment-api/assessment-actors/src/main/scala/org/sunbird/actors/QuestionSetActor.scala

@@ -4,8 +4,9 @@ import java.util
 
 import javax.inject.Inject
 import org.apache.commons.collections4.CollectionUtils
+import org.sunbird.`object`.importer.{ImportConfig, ImportManager}
 import org.sunbird.actor.core.BaseActor
-import org.sunbird.common.DateUtils
+import org.sunbird.common.{DateUtils, Platform}
 import org.sunbird.common.dto.{Request, Response, ResponseHandler}
 import org.sunbird.graph.OntologyEngineContext
 import org.sunbird.graph.nodes.DataNode
@@ -19,6 +20,8 @@ import scala.concurrent.{ExecutionContext, Future}
 class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends BaseActor {
 
 	implicit val ec: ExecutionContext = getContext().dispatcher
+	private lazy val importConfig = getImportConfig()
+	private lazy val importMgr = new ImportManager(importConfig)
 
 	override def onReceive(request: Request): Future[Response] = request.getOperation match {
 		case "createQuestionSet" => AssessmentManager.create(request, "ERR_QUESTION_SET_CREATE")
@@ -32,6 +35,7 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba
 		case "updateHierarchy" => UpdateHierarchyManager.updateHierarchy(request)
 		case "getHierarchy" => HierarchyManager.getHierarchy(request)
 		case "rejectQuestionSet" => reject(request)
+		case "importQuestionSet" => importQuestionSet(request)
 		case _ => ERROR(request.getOperation)
 	}
 
@@ -122,4 +126,15 @@ class QuestionSetActor @Inject()(implicit oec: OntologyEngineContext) extends Ba
 		})
 	}
 
+	def importQuestionSet(request: Request): Future[Response] = importMgr.importObject(request)
+
+	def getImportConfig(): ImportConfig = {
+		val requiredProps = Platform.getStringList("import.required_props.questionset", java.util.Arrays.asList("name", "code", "mimeType", "framework")).asScala.toList
+		val validStages = Platform.getStringList("import.valid_stages.questionset", java.util.Arrays.asList("create", "upload", "review", "publish")).asScala.toList
+		val propsToRemove = Platform.getStringList("import.remove_props.questionset", java.util.Arrays.asList()).asScala.toList
+		val topicName = Platform.config.getString("import.output_topic_name")
+		val reqLimit = Platform.getInteger("import.request_size_limit", 200)
+		ImportConfig(topicName, reqLimit, requiredProps, validStages, propsToRemove)
+	}
+
 }

assessment-api/assessment-actors/src/test/resources/application.conf

@@ -399,5 +399,18 @@ cassandra {
 }
 questionset.keyspace = "dev_hierarchy_store"
 
+import {
+  request_size_limit = 200
+  output_topic_name = "local.auto.creation.job.request"
+  required_props {
+    question = ["name", "code", "mimeType", "framework", "channel"]
+    questionset = ["name", "code", "mimeType", "framework", "channel"]
+  }
+  remove_props {
+    question = []
+    questionseet = []
+  }
+}
+
 
 

assessment-api/assessment-actors/src/test/scala/org/sunbird/actors/QuestionActorTest.scala

@@ -3,7 +3,8 @@ import java.util
 
 import akka.actor.Props
 import org.scalamock.scalatest.MockFactory
-import org.sunbird.common.dto.{Property, Request, Response}
+import org.sunbird.common.HttpUtil
+import org.sunbird.common.dto.{Property, Request, Response, ResponseHandler}
 import org.sunbird.graph.dac.model.Node
 import org.sunbird.graph.utils.ScalaJsonUtils
 import org.sunbird.graph.{GraphService, OntologyEngineContext}
@@ -118,6 +119,36 @@ class QuestionActorTest extends BaseSpec with MockFactory {
 		assert("successful".equals(response.getParams.getStatus))
 	}
 
+	it should "send events to kafka topic" in {
+		implicit val oec: OntologyEngineContext = mock[OntologyEngineContext]
+		val kfClient = mock[KafkaClient]
+		val hUtil = mock[HttpUtil]
+		(oec.httpUtil _).expects().returns(hUtil)
+		val resp :Response = ResponseHandler.OK()
+		resp.put("question", new util.HashMap[String, AnyRef](){{
+			put("framework", "NCF")
+			put("channel", "test")
+		}})
+		(hUtil.get(_: String, _: String, _: util.Map[String, String])).expects(*, *, *).returns(resp)
+		(oec.kafkaClient _).expects().returns(kfClient)
+		(kfClient.send(_: String, _: String)).expects(*, *).returns(None)
+		val request = getQuestionRequest()
+		request.getRequest.put("question", new util.HashMap[String, AnyRef](){{
+			put("source", "https://dock.sunbirded.org/api/question/v1/read/do_11307822356267827219477")
+			put("metadata", new util.HashMap[String, AnyRef](){{
+				put("name", "Test Question")
+				put("description", "Test Question")
+				put("mimeType", "application/vnd.sunbird.question")
+				put("code", "test.ques.1")
+				put("primaryCategory", "Learning Resource")
+			}})
+		}})
+		request.setOperation("importQuestion")
+		request.setObjectType("Question")
+		val response = callActor(request, Props(new QuestionActor()))
+		assert(response.get("processId") != null)
+	}
+
 	private def getQuestionRequest(): Request = {
 		val request = new Request()
 		request.setContext(new java.util.HashMap[String, AnyRef]() {

assessment-api/assessment-actors/src/test/scala/org/sunbird/actors/QuestionSetActorTest.scala

@@ -5,7 +5,8 @@ import java.util
 import akka.actor.Props
 import org.apache.commons.lang3.StringUtils
 import org.scalamock.scalatest.MockFactory
-import org.sunbird.common.dto.{Property, Request, Response}
+import org.sunbird.common.HttpUtil
+import org.sunbird.common.dto.{Property, Request, Response, ResponseHandler}
 import org.sunbird.graph.dac.model.{Node, Relation, SearchCriteria}
 import org.sunbird.graph.nodes.DataNode.getRelationMap
 import org.sunbird.graph.utils.ScalaJsonUtils
@@ -318,6 +319,36 @@ class QuestionSetActorTest extends BaseSpec with MockFactory {
         assert("successful".equals(response.getParams.getStatus))
     }
 
+    it should "send events to kafka topic" in {
+        implicit val oec: OntologyEngineContext = mock[OntologyEngineContext]
+        val kfClient = mock[KafkaClient]
+        val hUtil = mock[HttpUtil]
+        (oec.httpUtil _).expects().returns(hUtil)
+        val resp :Response = ResponseHandler.OK()
+        resp.put("questionset", new util.HashMap[String, AnyRef](){{
+            put("framework", "NCF")
+            put("channel", "test")
+        }})
+        (hUtil.get(_: String, _: String, _: util.Map[String, String])).expects(*, *, *).returns(resp)
+        (oec.kafkaClient _).expects().returns(kfClient)
+        (kfClient.send(_: String, _: String)).expects(*, *).returns(None)
+        val request = getQuestionSetRequest()
+        request.getRequest.put("questionset", new util.HashMap[String, AnyRef](){{
+            put("source", "https://dock.sunbirded.org/api/questionset/v1/read/do_11307822356267827219477")
+            put("metadata", new util.HashMap[String, AnyRef](){{
+                put("name", "Test QuestionSet")
+                put("description", "Test QuestionSet")
+                put("mimeType", "application/vnd.sunbird.questionset")
+                put("code", "test.ques.1")
+                put("primaryCategory", "Learning Resource")
+            }})
+        }})
+        request.setOperation("importQuestionSet")
+        request.setObjectType("QuestionSet")
+        val response = callActor(request, Props(new QuestionSetActor()))
+        assert(response.get("processId") != null)
+    }
+
     private def getQuestionSetRequest(): Request = {
         val request = new Request()
         request.setContext(new java.util.HashMap[String, AnyRef]() {

assessment-api/assessment-service/app/controllers/v4/QuestionController.scala

@@ -77,4 +77,13 @@ class QuestionController @Inject()(@Named(ActorNames.QUESTION_ACTOR) questionAct
 		questionRequest.getContext.put("identifier", identifier)
 		getResult(ApiId.RETIRE_QUESTION, questionActor, questionRequest)
 	}
+
+	def importQuestion() = Action.async { implicit request =>
+		val headers = commonHeaders()
+		val body = requestBody()
+		body.putAll(headers)
+		val questionRequest = getRequest(body, headers, QuestionOperations.importQuestion.toString)
+		setRequestContext(questionRequest, version, objectType, schemaName)
+		getResult(ApiId.IMPORT_QUESTION, questionActor, questionRequest)
+	}
 }

assessment-api/assessment-service/app/controllers/v4/QuestionSetController.scala

@@ -128,4 +128,13 @@ class QuestionSetController @Inject()(@Named(ActorNames.QUESTION_SET_ACTOR) ques
 		questionSetRequest.getContext.put("identifier", identifier)
 		getResult(ApiId.REJECT_QUESTION_SET, questionSetActor, questionSetRequest)
 	}
+
+	def importQuestionSet() = Action.async { implicit request =>
+		val headers = commonHeaders()
+		val body = requestBody()
+		body.putAll(headers)
+		val questionSetRequest = getRequest(body, headers, QuestionSetOperations.importQuestionSet.toString)
+		setRequestContext(questionSetRequest, version, objectType, schemaName)
+		getResult(ApiId.IMPORT_QUESTION_SET, questionSetActor, questionSetRequest)
+	}
 }

assessment-api/assessment-service/app/utils/ApiId.scala

@@ -19,6 +19,7 @@ object ApiId {
 	val REVIEW_QUESTION = "api.question.review"
 	val PUBLISH_QUESTION = "api.question.publish"
 	val RETIRE_QUESTION = "api.question.retire"
+	val IMPORT_QUESTION = "api.question.import"
 
 	//QuestionSet APIs
 	val CREATE_QUESTION_SET = "api.questionset.create"
@@ -32,5 +33,6 @@ object ApiId {
 	val UPDATE_HIERARCHY = "api.questionset.hierarchy.update"
 	val GET_HIERARCHY = "api.questionset.hierarchy.get"
 	val REJECT_QUESTION_SET = "api.questionset.reject"
+	val IMPORT_QUESTION_SET = "api.questionset.import"
 
 }

assessment-api/assessment-service/app/utils/QuestionOperations.scala

 package utils
 
 object QuestionOperations extends Enumeration {
-	val createQuestion, readQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion = Value
+	val createQuestion, readQuestion, updateQuestion, reviewQuestion, publishQuestion, retireQuestion, importQuestion = Value
 }

assessment-api/assessment-service/app/utils/QuestionSetOperations.scala

@@ -3,5 +3,5 @@ package utils
 object QuestionSetOperations extends Enumeration {
 	val createQuestionSet, readQuestionSet, updateQuestionSet, reviewQuestionSet, publishQuestionSet,
 	retireQuestionSet, addQuestion, removeQuestion, updateHierarchyQuestion, readHierarchyQuestion,
-	rejectQuestionSet = Value
+	rejectQuestionSet, importQuestionSet = Value
 }

assessment-api/assessment-service/conf/application.conf

@@ -390,4 +390,17 @@ cassandra {
     connection: "localhost:9042"
   }
 }
-neo4j_objecttypes_enabled=["Question"]
\ No newline at end of file
+neo4j_objecttypes_enabled=["Question"]
+
+import {
+  request_size_limit = 200
+  output_topic_name = "local.auto.creation.job.request"
+  required_props {
+    question = ["name", "code", "mimeType", "framework", "channel"]
+    questionseet = ["name", "code", "mimeType", "framework", "channel"]
+  }
+  remove_props {
+    question = []
+    questionseet = []
+  }
+}
\ No newline at end of file

assessment-api/assessment-service/conf/routes

@@ -18,6 +18,7 @@ PATCH     /question/v4/update/:identifier         controllers.v4.QuestionControl
 POST      /question/v4/review/:identifier         controllers.v4.QuestionController.review(identifier:String)
 POST      /question/v4/publish/:identifier        controllers.v4.QuestionController.publish(identifier:String)
 DELETE    /question/v4/retire/:identifier         controllers.v4.QuestionController.retire(identifier:String)
+POST     /question/v4/import                      controllers.v4.QuestionController.importQuestion()
 
 # QuestionSet API's
 POST      /questionset/v4/create                     controllers.v4.QuestionSetController.create
@@ -30,4 +31,5 @@ PATCH     /questionset/v4/add                        controllers.v4.QuestionSetC
 DELETE    /questionset/v4/remove                     controllers.v4.QuestionSetController.remove
 PATCH     /questionset/v4/hierarchy/update           controllers.v4.QuestionSetController.updateHierarchy
 GET       /questionset/v4/hierarchy/:identifier      controllers.v4.QuestionSetController.getHierarchy(identifier:String, mode:Option[String])
-POST      /questionset/v4/reject/:identifier         controllers.v4.QuestionSetController.reject(identifier:String)
\ No newline at end of file
+POST      /questionset/v4/reject/:identifier         controllers.v4.QuestionSetController.reject(identifier:String)
+POST     /questionset/v4/import                      controllers.v4.QuestionSetController.importQuestionSet()
\ No newline at end of file

assessment-api/assessment-service/test/controllers/v4/QuestionControllerSpec.scala

@@ -46,4 +46,10 @@ class QuestionControllerSpec extends BaseSpec {
 		isOK(result)
 		status(result)(defaultAwaitTimeout) must equalTo(OK)
 	}
+
+	"import should create a question successfully for given valid request" in {
+		val result = controller.importQuestion()(FakeRequest())
+		isOK(result)
+		status(result)(defaultAwaitTimeout) must equalTo(OK)
+	}
 }

assessment-api/assessment-service/test/controllers/v4/QuestionSetControllerSpec.scala

@@ -78,4 +78,10 @@ class QuestionSetControllerSpec extends BaseSpec {
 		isOK(result)
 		status(result)(defaultAwaitTimeout) must equalTo(OK)
 	}
+
+	"import should create an questionSet successfully for given valid request" in {
+		val result = controller.importQuestionSet()(FakeRequest())
+		isOK(result)
+		status(result)(defaultAwaitTimeout) must equalTo(OK)
+	}
 }

content-api/content-actors/pom.xml

@@ -38,6 +38,12 @@
             <version>1.0-SNAPSHOT</version>
             <type>jar</type>
         </dependency>
+        <dependency>
+            <groupId>org.sunbird</groupId>
+            <artifactId>import-manager</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.maj.version}</artifactId>

content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala

@@ -3,10 +3,11 @@ package org.sunbird.content.actors
 import java.util
 import java.util.concurrent.CompletionException
 import java.io.File
-import org.apache.commons.io.FilenameUtils
 
+import org.apache.commons.io.FilenameUtils
 import javax.inject.Inject
 import org.apache.commons.lang3.StringUtils
+import org.sunbird.`object`.importer.{ImportConfig, ImportManager}
 import org.sunbird.actor.core.BaseActor
 import org.sunbird.cache.impl.RedisCache
 import org.sunbird.content.util.{AcceptFlagManager, ContentConstants, CopyManager, DiscardManager, FlagManager, RetireManager}
@@ -15,7 +16,6 @@ import org.sunbird.common.{ContentParams, Platform, Slug}
 import org.sunbird.common.dto.{Request, Response, ResponseHandler}
 import org.sunbird.common.exception.ClientException
 import org.sunbird.content.dial.DIALManager
-import org.sunbird.content.mgr.ImportManager
 import org.sunbird.util.RequestUtil
 import org.sunbird.content.upload.mgr.UploadManager
 import org.sunbird.graph.OntologyEngineContext
@@ -24,11 +24,14 @@ import org.sunbird.graph.nodes.DataNode
 import org.sunbird.graph.utils.NodeUtil
 
 import scala.collection.JavaConverters
+import scala.collection.JavaConverters._
 import scala.concurrent.{ExecutionContext, Future}
 
 class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageService) extends BaseActor {
 
 	implicit val ec: ExecutionContext = getContext().dispatcher
+	private lazy val importConfig = getImportConfig()
+	private lazy val importMgr = new ImportManager(importConfig)
 
 	override def onReceive(request: Request): Future[Response] = {
 		request.getOperation match {
@@ -147,7 +150,7 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe
 
 	def linkDIALCode(request: Request): Future[Response] = DIALManager.link(request)
 
-	def importContent(request: Request): Future[Response] = ImportManager.importContent(request)
+	def importContent(request: Request): Future[Response] = importMgr.importObject(request)
 
 	def populateDefaultersForCreation(request: Request) = {
 		setDefaultsBasedOnMimeType(request, ContentParams.create.name)
@@ -202,4 +205,13 @@ class ContentActor @Inject() (implicit oec: OntologyEngineContext, ss: StorageSe
 		}
 		node
 	}
+
+	def getImportConfig(): ImportConfig = {
+		val requiredProps = Platform.getStringList("import.required_props", java.util.Arrays.asList("name", "code", "mimeType", "contentType", "artifactUrl", "framework")).asScala.toList
+		val validStages = Platform.getStringList("import.valid_stages", java.util.Arrays.asList("create", "upload", "review", "publish")).asScala.toList
+		val propsToRemove = Platform.getStringList("import.remove_props", java.util.Arrays.asList("downloadUrl", "variants", "previewUrl", "streamingUrl", "itemSets")).asScala.toList
+		val topicName = Platform.config.getString("import.output_topic_name")
+		val reqLimit = Platform.getInteger("import.request_size_limit", 200)
+		ImportConfig(topicName, reqLimit, requiredProps, validStages, propsToRemove)
+	}
 }

content-api/content-actors/src/test/resources/application.conf

@@ -503,7 +503,9 @@ content.link_dialcode.max_limit=10
 content.artifact.size.for_online=209715200
 
 # Import API Config
-content.import.request_size_limit=2
-content.import.required_props=["name","code","mimeType","contentType","artifactUrl","framework", "channel"]
-content.import.topic_name="local.auto.creation.job.request"
+import {
+  request_size_limit=2
+  output_topic_name="local.auto.creation.job.request"
+  required_props=["name","code","mimeType","contentType","artifactUrl","framework", "channel"]
+}
 channel.fetch.suggested_frameworks=false
\ No newline at end of file

content-api/content-actors/src/test/scala/org/sunbird/content/actors/TestContentActor.scala

@@ -344,6 +344,7 @@ class TestContentActor extends BaseSpec with MockFactory {
             }})
         }})
         request.setOperation("importContent")
+        request.setObjectType("Content")
         val response = callActor(request, Props(new ContentActor()))
         assert(response.get("processId") != null)
     }

content-api/content-service/conf/application.conf

@@ -586,9 +586,11 @@ content.link_dialcode.max_limit=10
 content.artifact.size.for_online=209715200
 
 # Content Import API Config
-content.import.request_size_limit=2
-content.import.required_props=["name","code","mimeType","contentType","artifactUrl","framework","channel"]
-content.import.topic_name="local.auto.creation.job.request"
+import {
+  request_size_limit=2
+  output_topic_name="local.auto.creation.job.request"
+  required_props=["name","code","mimeType","contentType","artifactUrl","framework", "channel"]
+}
 
 contentTypeToPrimaryCategory {
   ClassroomTeachingVideo: "Explanation Content"