feat: new command checks course version

Add command that checks the course version on Mongo and MySQL are
synchronized.
FAN-164
#40

nau_openedx_extensions/management/commands/courses_version_check_mongo_mysql.py

@@ -0,0 +1,228 @@
+"""
+Script that checks the course versions from MongoDB and MySQL.
+
+To manually develop the script you can edit it on the fly and execute it.
+docker cp courses_version_check_mongo_mysql.py \
+ openedx_lms:/openedx/venv/lib/python3.8/site-packages/nau_openedx_extensions/\
+ management/commands/courses_version_check_mongo_mysql.py && \
+ docker exec -i openedx_lms python manage.py lms courses_version_check_mongo_mysql
+"""
+
+from common.djangoapps.split_modulestore_django.models import ( # lint-amnesty, pylint: disable=import-error
+ SplitModulestoreCourseIndex,
+)
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from xmodule.mongo_utils import connect_to_mongodb # lint-amnesty, pylint: disable=import-error
+
+
+class Command(BaseCommand):
+ """
+ Script that checks the course versions from MongoDB and MySQL.
+ """
+
+ help = "Checks the course versions from MongoDB and MySQL"
+
+ def log_msg(self, msg):
+ self.stdout.write(str(msg))
+ self.stdout.flush()
+
+ @staticmethod
+ def organize_courses(mysql, mongo):
+ """
+ Function to check which courses are in mysql, mongo and both
+ """
+ mysql_courses = {course["course_id"] for course in mysql}
+ mongo_courses = {course["course_id"] for course in mongo}
+
+ # Courses that are only in mysql
+ only_mysql = [
+ course
+ for course in mysql
+ if course["course_id"] in mysql_courses - mongo_courses
+ ]
+ # Courses that are only in mysql
+ only_mongo = [
+ course
+ for course in mongo
+ if course["course_id"] in mongo_courses - mysql_courses
+ ]
+ # Courses that are in both databases
+ both = [
+ course["course_id"]
+ for course in mysql + mongo
+ if course["course_id"] in mysql_courses & mongo_courses
+ ]
+
+ return only_mysql, only_mongo, both
+
+ @staticmethod
+ def find_different_versions(mysql, mongo, id_list, mode):
+ """
+ Function to find courses that have different versions
+ """
+
+ different_versions = []
+ mysql_courses = [course for course in mysql if course["course_id"] in id_list]
+ mongo_courses = [course for course in mongo if course["course_id"] in id_list]
+
+ for mysql_course in mysql_courses:
+ for mongo_course in mongo_courses:
+ if (
+ mysql_course["course_id"] == mongo_course["course_id"]
+ and mysql_course[mode] != mongo_course[mode]
+ ):
+ different_versions.append(
+ {
+ "course_id": mysql_course["course_id"],
+ "mongo_id": mongo_course[mode],
+ "mongo_last_update": mongo_course["last_update"],
+ "mysql_id": mysql_course[mode],
+ "mysql_last_update": mysql_course["last_update"],
+ }
+ )
+
+ return different_versions
+
+ def handle(self, *args, **options):
+ """
+ Execute the command
+ """
+ # Configurations needed to connect to mongo
+ tz_aware = None
+ db = settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("db")
+ host = settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("host")
+ port = settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("port")
+ user = settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("user")
+ password = settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("password")
+
+ kwargs = {
+ "replicaSet": settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("replicaSet"),
+ "connectTimeoutMS": 2000,
+ "socketTimeoutMS": 3000,
+ "ssl": settings.CONTENTSTORE["DOC_STORE_CONFIG"].get("ssl"),
+ }
+
+ database = connect_to_mongodb(
+ db,
+ host,
+ port=port,
+ tz_aware=tz_aware,
+ user=user,
+ password=password,
+ **kwargs
+ )
+
+ active_versions = database["modulestore.active_versions"]
+
+ mongo_query = {}
+
+ # Take out all active versions
+ mongo_courses = list(active_versions.find(mongo_query))
+
+ mongo_publish = []
+ mongo_draft = []
+
+ # Keep only the information we need
+ for course in mongo_courses:
+ mongo_publish.append(
+ {
+ "publish_id": str(course.get("versions").get("published-branch")),
+ "course_id": "course-v1:"
+ + course.get("org")
+ + "+"
+ + course.get("course")
+ + "+"
+ + course.get("run"),
+ "last_update": course.get("last_update"),
+ }
+ )
+ mongo_draft.append(
+ {
+ "draft_id": str(course.get("versions").get("draft-branch")),
+ "course_id": "course-v1:"
+ + course.get("org")
+ + "+"
+ + course.get("course")
+ + "+"
+ + course.get("run"),
+ "last_update": course.get("last_update"),
+ }
+ )
+
+ # Take out all active versions
+ mysql_courses = SplitModulestoreCourseIndex.objects.all()
+
+ mysql_courses = list(
+ SplitModulestoreCourseIndex.objects.values(
+ "course_id",
+ "draft_version",
+ "published_version",
+ "library_version",
+ "last_update",
+ )
+ )
+
+ mysql_publish = []
+ mysql_draft = []
+
+ # Keep only the information we need
+ for course in mysql_courses:
+ mysql_publish.append(
+ {
+ "publish_id": course["published_version"],
+ "course_id": str(course["course_id"]),
+ "last_update": course["last_update"],
+ }
+ )
+ mysql_draft.append(
+ {
+ "draft_id": course["draft_version"],
+ "course_id": str(course["course_id"]),
+ "last_update": course["last_update"],
+ }
+ )
+
+ publish_only_mysql, publish_only_mongo, publish_both = self.organize_courses(
+ mysql_publish, mongo_publish
+ )
+
+ self.log_msg("=" * 100)
+ self.log_msg("Publish version only in mysql")
+ self.log_msg("=" * 100)
+ self.log_msg(publish_only_mysql)
+ self.log_msg("=" * 100)
+ self.log_msg("Draft version only in mongo")
+ self.log_msg("=" * 100)
+ self.log_msg(publish_only_mongo)
+
+ draft_only_mysql, draft_only_mongo, draft_both = self.organize_courses(
+ mysql_draft, mongo_draft
+ )
+
+ self.log_msg("=" * 100)
+ self.log_msg("Draft version only in mysql")
+ self.log_msg("=" * 100)
+ self.log_msg(draft_only_mysql)
+ self.log_msg("=" * 100)
+ self.log_msg("Draft version only in mongo")
+ self.log_msg("=" * 100)
+ self.log_msg(draft_only_mongo)
+
+ different_publish_versions = self.find_different_versions(
+ mysql_publish, mongo_publish, publish_both, "publish_id"
+ )
+
+ self.log_msg("=" * 100)
+ self.log_msg("Different publish version")
+ self.log_msg("=" * 100)
+ self.log_msg(different_publish_versions)
+
+ different_draft_versions = self.find_different_versions(
+ mysql_draft, mongo_draft, draft_both, "draft_id"
+ )
+
+ self.log_msg("=" * 100)
+ self.log_msg("Different draft version")
+ self.log_msg("=" * 100)
+ self.log_msg(different_draft_versions)