From 2831f33587d381d21b605a0738161bf720f95ee9 Mon Sep 17 00:00:00 2001 From: Kushagra Tiwari Date: Wed, 18 Feb 2026 20:24:26 +0530 Subject: [PATCH 1/4] Implement caching for homepage data and signals to improve performance - Added caching for categories, recent questions, active questions, slider questions, spam questions, and category question mapping in views. - Introduced a cache invalidation mechanism in signals for questions and answers. - Updated the settings to include cache configuration. - Enhanced template tags to cache total question and answer counts. - Refactored the category image retrieval to utilize caching. --- forums/settings.py | 7 +- website/apps.py | 55 +++++++++- website/signals.py | 27 ++++- website/templatetags/count_tags.py | 13 +++ website/templatetags/helpers.py | 15 ++- website/views.py | 164 +++++++++++++++++++++-------- 6 files changed, 227 insertions(+), 54 deletions(-) diff --git a/forums/settings.py b/forums/settings.py index 756c88f..aa0c947 100644 --- a/forums/settings.py +++ b/forums/settings.py @@ -235,12 +235,13 @@ 'compressor.filters.css_default.CssAbsoluteFilter', 'compressor.filters.cssmin.CSSMinFilter', ) -"""CACHES = { + +CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake' + 'LOCATION': 'unique-snowflake', } -}""" +} COMPRESS_ENABLED = True diff --git a/website/apps.py b/website/apps.py index f1af676..f55fedb 100644 --- a/website/apps.py +++ b/website/apps.py @@ -1,11 +1,56 @@ from django.apps import AppConfig -from django.db.models.signals import post_save +from django.db.models.signals import post_save, post_delete + class WebsiteConfig(AppConfig): name = 'website' def ready(self): - from .models import Answer, AnswerComment - from .signals import last_active_signal_from_answer, last_active_signal_from_reply - post_save.connect(last_active_signal_from_answer, sender=Answer, dispatch_uid='trigger_last_active_answer') - post_save.connect(last_active_signal_from_reply, sender=AnswerComment, dispatch_uid='trigger_last_active_reply') \ No newline at end of file + from .models import Question, Answer, AnswerComment + from .signals import ( + last_active_signal_from_answer, + last_active_signal_from_reply, + home_cache_invalidator, + ) + + post_save.connect( + last_active_signal_from_answer, + sender=Answer, + dispatch_uid='trigger_last_active_answer', + ) + post_save.connect( + last_active_signal_from_reply, + sender=AnswerComment, + dispatch_uid='trigger_last_active_reply', + ) + + post_save.connect( + home_cache_invalidator, + sender=Question, + dispatch_uid='home_cache_invalidator_question_save', + ) + post_delete.connect( + home_cache_invalidator, + sender=Question, + dispatch_uid='home_cache_invalidator_question_delete', + ) + post_save.connect( + home_cache_invalidator, + sender=Answer, + dispatch_uid='home_cache_invalidator_answer_save', + ) + post_delete.connect( + home_cache_invalidator, + sender=Answer, + dispatch_uid='home_cache_invalidator_answer_delete', + ) + post_save.connect( + home_cache_invalidator, + sender=AnswerComment, + dispatch_uid='home_cache_invalidator_answercomment_save', + ) + post_delete.connect( + home_cache_invalidator, + sender=AnswerComment, + dispatch_uid='home_cache_invalidator_answercomment_delete', + ) \ No newline at end of file diff --git a/website/signals.py b/website/signals.py index 42dd479..809f49d 100644 --- a/website/signals.py +++ b/website/signals.py @@ -1,13 +1,38 @@ from django.utils import timezone +from django.core.cache import cache + + +HOME_CACHE_KEYS = [ + 'home:categories', + 'home:recent_questions', + 'home:active_questions', + 'home:slider_questions', + 'home:spam_questions', + 'home:category_question_map', + 'stats:total_questions', + 'stats:total_answers', +] + + +def clear_home_cache(): + cache.delete_many(HOME_CACHE_KEYS) + def last_active_signal_from_answer(sender, instance, created, **kwargs): if created or not created: instance.question.last_active = timezone.now() instance.question.last_post_by = instance.uid instance.question.save() + clear_home_cache() + def last_active_signal_from_reply(sender, instance, created, **kwargs): if created or not created: instance.answer.question.last_active = timezone.now() instance.answer.question.last_post_by = instance.uid - instance.answer.question.save() \ No newline at end of file + instance.answer.question.save() + clear_home_cache() + + +def home_cache_invalidator(sender, instance, **kwargs): + clear_home_cache() \ No newline at end of file diff --git a/website/templatetags/count_tags.py b/website/templatetags/count_tags.py index 4dc603e..f5c3531 100755 --- a/website/templatetags/count_tags.py +++ b/website/templatetags/count_tags.py @@ -1,4 +1,5 @@ from django import template +from django.core.cache import cache from website.models import Question, Answer @@ -86,7 +87,13 @@ def div(value, arg=1): # retriving total number of questions def total_question_count(): + cache_key = 'stats:total_questions' + count = cache.get(cache_key) + if count is not None: + return count + count = Question.objects.filter(status=1).count() + cache.set(cache_key, count, 10) return count @@ -95,7 +102,13 @@ def total_question_count(): # retriving total number of answers def total_answer_count(): + cache_key = 'stats:total_answers' + count = cache.get(cache_key) + if count is not None: + return count + count = Answer.objects.filter(question__status=1).count() + cache.set(cache_key, count, 10) return count diff --git a/website/templatetags/helpers.py b/website/templatetags/helpers.py index 5f85d91..70ffb35 100755 --- a/website/templatetags/helpers.py +++ b/website/templatetags/helpers.py @@ -1,6 +1,6 @@ from django import template +from django.core.cache import cache -# from website.models import Question, Answer, Notification from website.helpers import prettify from django.conf import settings import os.path @@ -9,12 +9,21 @@ def get_category_image(category): + cache_key = f'category_image:{category}' + cached = cache.get(cache_key) + if cached is not None: + return cached + base_path = settings.BASE_DIR + '/static/website/images/' file_name = category.replace(' ', '-') + '.jpg' file_path = base_path + file_name if os.path.isfile(file_path): - return 'website/images/' + file_name - return False + value = 'website/images/' + file_name + else: + value = False + + cache.set(cache_key, value, 3600) + return value register.filter('get_category_image', get_category_image) diff --git a/website/views.py b/website/views.py index 9f4f6df..3232008 100755 --- a/website/views.py +++ b/website/views.py @@ -10,6 +10,7 @@ from django.core.mail import EmailMultiAlternatives from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.contrib.auth import get_user_model +from django.core.cache import cache from website.models import Question, Answer, Notification, AnswerComment from spoken_auth.models import TutorialDetails, TutorialResources @@ -23,31 +24,130 @@ from website.permissions import is_administrator, is_forumsadmin User = get_user_model() -categories = [] -trs = TutorialResources.objects.filter(Q(status=1) | Q(status=2),tutorial_detail__foss__show_on_homepage__lt=2, language__name='English') -trs = trs.values('tutorial_detail__foss__foss').order_by('tutorial_detail__foss__foss') -for tr in trs.values_list('tutorial_detail__foss__foss').distinct(): - categories.append(tr[0]) +HOME_CACHE_TIMEOUT = 10 + + +def _get_home_categories(): + cache_key = 'home:categories' + categories = cache.get(cache_key) + if categories is not None: + return categories + + categories = [] + trs = TutorialResources.objects.filter( + Q(status=1) | Q(status=2), + tutorial_detail__foss__show_on_homepage__lt=2, + language__name='English', + ) + trs = trs.values_list('tutorial_detail__foss__foss', flat=True).order_by('tutorial_detail__foss__foss').distinct() + for foss_name in trs: + categories.append(foss_name) + + cache.set(cache_key, categories, HOME_CACHE_TIMEOUT) + return categories + + +def _get_home_base_queryset(): + return Question.objects.annotate(answer_count=Count('answer')) + + +def _get_home_questions(): + cache_key = 'home:recent_questions' + questions = cache.get(cache_key) + if questions is not None: + return questions + base_queryset = _get_home_base_queryset() + questions = list(base_queryset.filter(status=1).order_by('-date_created')[:100]) + cache.set(cache_key, questions, HOME_CACHE_TIMEOUT) + return questions + + +def _get_home_active_questions(): + cache_key = 'home:active_questions' + questions = cache.get(cache_key) + if questions is not None: + return questions + base_queryset = _get_home_base_queryset() + questions = list( + base_queryset.filter(status=1, last_active__isnull=False).order_by('-last_active')[:100] + ) + cache.set(cache_key, questions, HOME_CACHE_TIMEOUT) + return questions + + +def _get_home_slider_questions(): + cache_key = 'home:slider_questions' + slider_questions = cache.get(cache_key) + if slider_questions is not None: + return slider_questions + + base_queryset = _get_home_base_queryset() + subquery = ( + Question.objects.filter(category=OuterRef('category'), status=1) + .values('category') + .annotate(max_date=Max('date_created')) + .values('max_date') + ) + slider_questions = list( + base_queryset.filter(date_created=Subquery(subquery), status=1).order_by('category') + ) + cache.set(cache_key, slider_questions, HOME_CACHE_TIMEOUT) + return slider_questions + + +def _get_home_spam_questions(): + cache_key = 'home:spam_questions' + spam_questions = cache.get(cache_key) + if spam_questions is not None: + return spam_questions + base_queryset = _get_home_base_queryset() + spam_questions = list(base_queryset.filter(status=2).order_by('-last_active')[:100]) + cache.set(cache_key, spam_questions, HOME_CACHE_TIMEOUT) + return spam_questions + + +def _get_home_category_question_map(categories, slider_questions): + cache_key = 'home:category_question_map' + category_question_map = cache.get(cache_key) + if category_question_map is not None: + return category_question_map + + category_fosses = {val.replace(" ", "-"): val for val in categories} + all_eligible_categories = list(category_fosses.keys()) + + category_question_map = {} + for question in slider_questions: + if question.category in all_eligible_categories: + foss = category_fosses.get(question.category) + category_question_map[foss] = { + "id": question.id, + "question": question.title, + "foss_url": question.category, + } + + for category in category_fosses.keys(): + foss = category_fosses.get(category) + if foss not in category_question_map: + category_question_map[foss] = None + + category_question_map = dict( + sorted(category_question_map.items(), key=lambda item: item[0].lower()) + ) + cache.set(cache_key, category_question_map, HOME_CACHE_TIMEOUT) + return category_question_map def home(request): - # Base query with answer count annotation - base_queryset = Question.objects.annotate(answer_count=Count('answer')) - - questions = base_queryset.filter(status=1).order_by('-date_created')[:100] - active_questions = base_queryset.filter(status=1, last_active__isnull=False).order_by('-last_active')[:100] - - # Retrieve latest questions per category for the slider - subquery = Question.objects.filter(category=OuterRef('category'), status=1).values('category').annotate(max_date=Max('date_created')).values('max_date') - slider_questions = base_queryset.filter( - date_created=Subquery(subquery), status=1 - ).order_by('category') - - # spam questions - spam_questions = base_queryset.filter(status=2).order_by('-last_active')[:100] + questions = _get_home_questions() + active_questions = _get_home_active_questions() + slider_questions = _get_home_slider_questions() + + spam_questions = [] + show_spam_list = is_administrator(request.user) or is_forumsadmin(request.user) + if show_spam_list: + spam_questions = _get_home_spam_questions() - # Bulk fetch users for all displayed questions to avoid N+1 all_questions = list(questions) + list(active_questions) + list(slider_questions) + list(spam_questions) uids = set() for q in all_questions: @@ -62,28 +162,8 @@ def home(request): q.cached_user = users.get(q.uid, "Unknown User") q.cached_last_post_user = users.get(q.last_post_by, "Unknown User") if q.last_post_by else "Unknown User" - # Mapping of foss name as in spk db & its corresponding category name in forums db - category_fosses = {val.replace(" ", "-") : val for val in categories} - - # All eligible categories to be shown in homepage slider - all_eligible_categories = list(category_fosses.keys()) - - # Create a dictionary to map categories to questions for the slider - category_question_map = {} - for question in slider_questions: - if question.category in all_eligible_categories: - foss = category_fosses.get(question.category) - category_question_map[foss] = {"id" : question.id, "question": question.title, "foss_url": question.category} - - # Fill in missing categories without questions - for category in category_fosses.keys(): - foss = category_fosses.get(category) - if foss not in category_question_map: - category_question_map[foss] = None - - # Sort category_question_map by category name - category_question_map = dict(sorted(category_question_map.items(), key= lambda item: item[0].lower())) - show_spam_list = is_administrator(request.user) or is_forumsadmin(request.user) + categories = _get_home_categories() + category_question_map = _get_home_category_question_map(categories, slider_questions) context = { 'questions': questions, From 55973c5316e9ec859a95eb6ac8b4728a3bb92697 Mon Sep 17 00:00:00 2001 From: Kushagra Tiwari Date: Thu, 19 Feb 2026 18:48:35 +0530 Subject: [PATCH 2/4] Update caching configuration and improve homepage data retrieval - Changed cache backend to Memcached and added a file cache option in settings. - Increased HOME_CACHE_TIMEOUT to 3600 seconds for better performance. - Refactored homepage data retrieval functions to accept a base queryset, enhancing flexibility and efficiency. - Updated calls to caching functions to utilize the new base queryset parameter. --- forums/settings.py | 13 ++++++++++--- website/views.py | 35 ++++++++++++++--------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/forums/settings.py b/forums/settings.py index aa0c947..1f4a03a 100644 --- a/forums/settings.py +++ b/forums/settings.py @@ -238,9 +238,16 @@ CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - } + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': 'localhost:11211', + 'TIMEOUT': 3600 * 24, + 'KEY_PREFIX': 'forums', + }, + 'file_cache': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'topper_cache_table', + 'TIMEOUT': 3600 * 24 * 30, + }, } COMPRESS_ENABLED = True diff --git a/website/views.py b/website/views.py index 3232008..62f8bd4 100755 --- a/website/views.py +++ b/website/views.py @@ -25,7 +25,7 @@ User = get_user_model() -HOME_CACHE_TIMEOUT = 10 +HOME_CACHE_TIMEOUT = 3600 def _get_home_categories(): @@ -34,41 +34,32 @@ def _get_home_categories(): if categories is not None: return categories - categories = [] trs = TutorialResources.objects.filter( Q(status=1) | Q(status=2), tutorial_detail__foss__show_on_homepage__lt=2, language__name='English', ) trs = trs.values_list('tutorial_detail__foss__foss', flat=True).order_by('tutorial_detail__foss__foss').distinct() - for foss_name in trs: - categories.append(foss_name) - + categories = list(trs) cache.set(cache_key, categories, HOME_CACHE_TIMEOUT) return categories -def _get_home_base_queryset(): - return Question.objects.annotate(answer_count=Count('answer')) - - -def _get_home_questions(): +def _get_home_questions(base_queryset): cache_key = 'home:recent_questions' questions = cache.get(cache_key) if questions is not None: return questions - base_queryset = _get_home_base_queryset() questions = list(base_queryset.filter(status=1).order_by('-date_created')[:100]) cache.set(cache_key, questions, HOME_CACHE_TIMEOUT) return questions -def _get_home_active_questions(): +def _get_home_active_questions(base_queryset): cache_key = 'home:active_questions' questions = cache.get(cache_key) if questions is not None: return questions - base_queryset = _get_home_base_queryset() questions = list( base_queryset.filter(status=1, last_active__isnull=False).order_by('-last_active')[:100] ) @@ -76,13 +67,12 @@ def _get_home_active_questions(): return questions -def _get_home_slider_questions(): +def _get_home_slider_questions(base_queryset): cache_key = 'home:slider_questions' slider_questions = cache.get(cache_key) if slider_questions is not None: return slider_questions - base_queryset = _get_home_base_queryset() subquery = ( Question.objects.filter(category=OuterRef('category'), status=1) .values('category') @@ -96,12 +86,11 @@ def _get_home_slider_questions(): return slider_questions -def _get_home_spam_questions(): +def _get_home_spam_questions(base_queryset): cache_key = 'home:spam_questions' spam_questions = cache.get(cache_key) if spam_questions is not None: return spam_questions - base_queryset = _get_home_base_queryset() spam_questions = list(base_queryset.filter(status=2).order_by('-last_active')[:100]) cache.set(cache_key, spam_questions, HOME_CACHE_TIMEOUT) return spam_questions @@ -139,14 +128,16 @@ def _get_home_category_question_map(categories, slider_questions): def home(request): - questions = _get_home_questions() - active_questions = _get_home_active_questions() - slider_questions = _get_home_slider_questions() + base_queryset = Question.objects.annotate(answer_count=Count('answer')) + + questions = _get_home_questions(base_queryset) + active_questions = _get_home_active_questions(base_queryset) + slider_questions = _get_home_slider_questions(base_queryset) spam_questions = [] show_spam_list = is_administrator(request.user) or is_forumsadmin(request.user) if show_spam_list: - spam_questions = _get_home_spam_questions() + spam_questions = _get_home_spam_questions(base_queryset) all_questions = list(questions) + list(active_questions) + list(slider_questions) + list(spam_questions) uids = set() @@ -694,6 +685,7 @@ def clear_notifications(request): def search(request): + categories = _get_home_categories() context = { 'categories': categories } @@ -704,6 +696,7 @@ def search(request): def ajax_category(request): + categories = _get_home_categories() context = { 'categories': categories } From 1551f1ff504f531345757bbba1caadcd604d57fa Mon Sep 17 00:00:00 2001 From: Kushagra Tiwari Date: Fri, 20 Feb 2026 10:59:41 +0530 Subject: [PATCH 3/4] Refactor caching configuration to support Memcached and local memory fallback - Updated the caching settings to use Memcached if available, otherwise default to local memory for development. - Simplified cache backend configuration by defining a single variable for the default cache settings. --- forums/settings.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/forums/settings.py b/forums/settings.py index 1f4a03a..b917c6a 100644 --- a/forums/settings.py +++ b/forums/settings.py @@ -236,13 +236,25 @@ 'compressor.filters.cssmin.CSSMinFilter', ) -CACHES = { - 'default': { +# Use Memcached when the memcache package is available; otherwise use local memory +# (e.g. for local development without Memcached). +try: + import memcache # noqa: F401 + _default_cache = { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': 'localhost:11211', 'TIMEOUT': 3600 * 24, 'KEY_PREFIX': 'forums', - }, + } +except ImportError: + _default_cache = { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'TIMEOUT': 3600 * 24, + 'KEY_PREFIX': 'forums', + } + +CACHES = { + 'default': _default_cache, 'file_cache': { 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'topper_cache_table', From 0bce40c7731d88d61c0f4ac0da3911ffb359ff0c Mon Sep 17 00:00:00 2001 From: Kushagra Tiwari Date: Thu, 5 Mar 2026 20:13:10 +0530 Subject: [PATCH 4/4] Enhance user data caching and template rendering for performance improvements - Updated templates to utilize cached user data for questions, answers, and notifications, reducing database queries. - Refactored views to attach cached usernames to questions and answers, improving efficiency in rendering. - Implemented caching for question counts in template tags to speed up category data retrieval. - Adjusted notification rendering to leverage cached attributes, minimizing additional database lookups. --- .../templates/ajax-keyword-search.html | 8 +- .../website/templates/ajax-time-search.html | 8 +- static/website/templates/filter.html | 12 +- static/website/templates/get-question.html | 6 +- static/website/templates/notifications.html | 2 +- static/website/templates/questions.html | 8 +- .../website/templates/recent-questions.html | 17 +- website/models.py | 35 +++- website/templatetags/count_tags.py | 12 +- website/templatetags/notify.py | 20 +- website/templatetags/sidebar_tags.py | 15 +- website/views.py | 172 +++++++++++++----- 12 files changed, 218 insertions(+), 97 deletions(-) diff --git a/static/website/templates/ajax-keyword-search.html b/static/website/templates/ajax-keyword-search.html index b144529..68cb41d 100755 --- a/static/website/templates/ajax-keyword-search.html +++ b/static/website/templates/ajax-keyword-search.html @@ -63,16 +63,16 @@ - {{ question.views}} + {{ question.views }} - {{ question.answer_set.count }} + {{ question.total_answers }} - - {{ question.user|truncatechars:10 }} + + {{ question.cached_user|default:question.user|truncatechars:10 }} diff --git a/static/website/templates/ajax-time-search.html b/static/website/templates/ajax-time-search.html index 57be751..c26e4b7 100755 --- a/static/website/templates/ajax-time-search.html +++ b/static/website/templates/ajax-time-search.html @@ -61,16 +61,16 @@ - {{ question.views}} + {{ question.views }} - {{ question.answer_set.count }} + {{ question.total_answers }} - - {{ question.user|truncatechars:10 }} + + {{ question.cached_user|default:question.user|truncatechars:10 }} diff --git a/static/website/templates/filter.html b/static/website/templates/filter.html index 680fe28..f1b6e62 100755 --- a/static/website/templates/filter.html +++ b/static/website/templates/filter.html @@ -82,16 +82,16 @@
- {{ question.views}} + {{ question.views }} - + - {{ question.answer_set.count }} + {{ question.total_answers }} - + - - {{ question.user|truncatechars:10 }} + + {{ question.cached_user|default:question.user|truncatechars:10 }} diff --git a/static/website/templates/get-question.html b/static/website/templates/get-question.html index 7bc8bc0..1348ab5 100755 --- a/static/website/templates/get-question.html +++ b/static/website/templates/get-question.html @@ -136,7 +136,7 @@
Question
- {{ question.user }} + {{ question.cached_user|default:question.user }} @@ -169,7 +169,7 @@

Answers:

- {{ answer.user }} + {{ answer.cached_user|default:answer.user }} {% if user|can_edit:answer %} @@ -199,7 +199,7 @@

Answers:

- {{ comment.user }} + {{ comment.cached_user|default:comment.user }} {% if user|can_edit:comment %} diff --git a/static/website/templates/notifications.html b/static/website/templates/notifications.html index aa3adce..68f833f 100755 --- a/static/website/templates/notifications.html +++ b/static/website/templates/notifications.html @@ -7,7 +7,7 @@

Notifications

{% for notification in notifications %} - {% get_notification notification.id %} + {% get_notification notification %} {% endfor %} {% endblock %} diff --git a/static/website/templates/questions.html b/static/website/templates/questions.html index cfb0c9e..5907aec 100755 --- a/static/website/templates/questions.html +++ b/static/website/templates/questions.html @@ -68,16 +68,16 @@
- {{ question.views}} + {{ question.views }} - {{ question.answer_set.count }} + {{ question.total_answers }} - - {{ question.user|truncatechars:10 }} + + {{ question.cached_user|default:question.user|truncatechars:10 }} diff --git a/static/website/templates/recent-questions.html b/static/website/templates/recent-questions.html index 3461908..ed80301 100755 --- a/static/website/templates/recent-questions.html +++ b/static/website/templates/recent-questions.html @@ -1,9 +1,7 @@ -{% extends 'website/templates/base.html' %} {% load static %} {% load count_tags %} -{% block content %} -

Recent Questions

+

Recent Questions

@@ -65,16 +63,16 @@

Recent Questions

@@ -96,11 +94,6 @@

Recent Questions

{% endfor %} {% endif %} - -{% endblock %} - -{% block javascript %} -{% endblock %} diff --git a/website/models.py b/website/models.py index cd9040e..b45e2e9 100755 --- a/website/models.py +++ b/website/models.py @@ -21,12 +21,22 @@ class Question(models.Model): # votes = models.IntegerField(default=0) def user(self): + cached_username = getattr(self, "cached_user", None) + if cached_username is not None: + return cached_username user = User.objects.get(id=self.uid) - return user.username + username = user.username + self.cached_user = username + return username def last_post_user(self): + cached_username = getattr(self, "cached_last_post_user", None) + if cached_username is not None: + return cached_username user = User.objects.filter(id=self.last_post_by).first() - return user.username if user else "Unknown User" + username = user.username if user else "Unknown User" + self.cached_last_post_user = username + return username class Meta: get_latest_by = "date_created" @@ -54,8 +64,13 @@ class Answer(models.Model): # votes = models.IntegerField(default=0) def user(self): + cached_username = getattr(self, "cached_user", None) + if cached_username is not None: + return cached_username user = User.objects.get(id=self.uid) - return user.username + username = user.username + self.cached_user = username + return username class AnswerVote(models.Model): @@ -71,8 +86,13 @@ class AnswerComment(models.Model): date_modified = models.DateTimeField(auto_now=True) def user(self): + cached_username = getattr(self, "cached_user", None) + if cached_username is not None: + return cached_username user = User.objects.get(id=self.uid) - return user.username + username = user.username + self.cached_user = username + return username class Notification(models.Model): @@ -84,7 +104,12 @@ class Notification(models.Model): date_created = models.DateTimeField(auto_now_add=True) def poster(self): + cached_username = getattr(self, "cached_poster", None) + if cached_username is not None: + return cached_username user = User.objects.get(id=self.pid) - return user.username + username = user.username + self.cached_poster = username + return username # CDEEP database created using inspectdb arg of manage.py diff --git a/website/templatetags/count_tags.py b/website/templatetags/count_tags.py index f5c3531..0f70862 100755 --- a/website/templatetags/count_tags.py +++ b/website/templatetags/count_tags.py @@ -6,10 +6,16 @@ register = template.Library() -# Counts the number of questions in +# Counts the number of questions in , cached for faster rendering def category_question_count(category): - category_question_count = Question.objects.filter(category=category).count() - return category_question_count + cache_key = f"stats:category_question_count:{category}" + count = cache.get(cache_key) + if count is not None: + return count + + count = Question.objects.filter(category=category).count() + cache.set(cache_key, count, 300) + return count register.simple_tag(category_question_count) diff --git a/website/templatetags/notify.py b/website/templatetags/notify.py index 8da07f7..c2b4144 100755 --- a/website/templatetags/notify.py +++ b/website/templatetags/notify.py @@ -5,16 +5,16 @@ register = template.Library() -def get_notification(nid): - notification = Notification.objects.get(pk=nid) - try: - question = Question.objects.get(pk=notification.qid) - except Question.DoesNotExist: - question = None - try: - answer = Answer.objects.get(pk=notification.aid) - except Answer.DoesNotExist: - answer = None +def get_notification(notification): + """ + Render a single notification. + + The view (`user_notifications`) attaches `cached_question`, `cached_answer` + and `cached_poster` attributes on each `Notification` instance so that this + tag does not need to perform any additional database queries. + """ + question = getattr(notification, "cached_question", None) + answer = getattr(notification, "cached_answer", None) context = { 'notification': notification, 'question': question, diff --git a/website/templatetags/sidebar_tags.py b/website/templatetags/sidebar_tags.py index a175052..e000888 100755 --- a/website/templatetags/sidebar_tags.py +++ b/website/templatetags/sidebar_tags.py @@ -1,4 +1,5 @@ from django import template +from django.db.models import Count from website.models import Question @@ -6,8 +7,16 @@ def recent_questions(): - recent_questions = Question.objects.all().order_by('-id')[:5] - return {'recent_questions': recent_questions} + recent_questions = ( + Question.objects.all() + .annotate(total_answers=Count('answer')) + .order_by('-id')[:5] + ) + return { + 'questions': recent_questions, + 'total': 10, + 'marker': 0, + } -register.inclusion_tag('website/templates/recent_questions.html')(recent_questions) +register.inclusion_tag('website/templates/recent-questions.html')(recent_questions) diff --git a/website/views.py b/website/views.py index 62f8bd4..01b0116 100755 --- a/website/views.py +++ b/website/views.py @@ -167,8 +167,7 @@ def home(request): def questions(request): - questions = Question.objects.filter(status=1).order_by('category', 'tutorial') - questions = questions.annotate(total_answers=Count('answer')) + questions = Question.objects.filter(status=1).annotate(total_answers=Count('answer')).order_by('category', 'tutorial') raw_get_data = request.GET.get('o', None) @@ -194,16 +193,22 @@ def questions(request): questions = paginator.page(1) except EmptyPage: questions = paginator.page(paginator.num_pages) + # Attach cached usernames to avoid N+1 in templates + uids = {q.uid for q in questions} + users = {u.id: u.username for u in User.objects.filter(id__in=uids)} + for q in questions: + q.cached_user = users.get(q.uid, "Unknown User") + context = { 'questions': questions, 'header': header, - 'ordering': ordering - } + 'ordering': ordering, + } return render(request, 'website/templates/questions.html', context) def hidden_questions(request): - questions = Question.objects.filter(status=0).order_by('date_created').reverse() + questions = Question.objects.filter(status=0).annotate(total_answers=Count('answer')).order_by('-date_created') paginator = Paginator(questions, 20) page = request.GET.get('page') @@ -213,8 +218,14 @@ def hidden_questions(request): questions = paginator.page(1) except EmptyPage: questions = paginator.page(paginator.num_pages) + # Attach cached usernames similar to questions() view + uids = {q.uid for q in questions} + users = {u.id: u.username for u in User.objects.filter(id__in=uids)} + for q in questions: + q.cached_user = users.get(q.uid, "Unknown User") + context = { - 'questions': questions + 'questions': questions, } return render(request, 'website/templates/questions.html', context) @@ -225,19 +236,39 @@ def get_question(request, question_id=None, pretty_url=None): category = FossCategory.objects.all().order_by('foss') if pretty_url != pretty_title: return HttpResponseRedirect('/question/' + question_id + '/' + pretty_title) - answers = question.answer_set.all() + # Prefetch answers and their comments to avoid N+1 queries + answers = ( + question.answer_set.all() + .prefetch_related('answercomment_set') + .order_by('date_created') + ) form = AnswerQuesitionForm() if question.status in (0,2): label = "Show" else: label = "Hide" + # Cache usernames for question, answers and comments + uids = {question.uid} + for answer in answers: + uids.add(answer.uid) + for comment in answer.answercomment_set.all(): + uids.add(comment.uid) + + users = {u.id: u.username for u in User.objects.filter(id__in=uids)} + + question.cached_user = users.get(question.uid, "Unknown User") + for answer in answers: + answer.cached_user = users.get(answer.uid, "Unknown User") + for comment in answer.answercomment_set.all(): + comment.cached_user = users.get(comment.uid, "Unknown User") + context = { 'question': question, 'answers': answers, 'category': category, 'form': form, - 'label': label + 'label': label, } user_has_role = has_role(request.user) context['require_recaptcha'] = not user_has_role @@ -657,7 +688,11 @@ def user_answers(request, user_id): if str(user_id) == str(request.user.id): total = Answer.objects.filter(uid=user_id).count() total = int(total - (total % 10 - 10)) - answers = Answer.objects.filter(uid=user_id).order_by('date_created').reverse()[marker:marker + 10] + answers = ( + Answer.objects.filter(uid=user_id) + .select_related('question') + .order_by('-date_created')[marker:marker + 10] + ) context = { 'answers': answers, 'total': total, @@ -670,9 +705,26 @@ def user_answers(request, user_id): @login_required def user_notifications(request, user_id): if str(user_id) == str(request.user.id): - notifications = Notification.objects.filter(uid=user_id).order_by('date_created').reverse() + notifications = list( + Notification.objects.filter(uid=user_id).order_by('-date_created') + ) + + # Prefetch related questions, answers and posters in bulk + qids = {n.qid for n in notifications if n.qid} + aids = {n.aid for n in notifications if n.aid} + pids = {n.pid for n in notifications if n.pid} + + questions = {q.id: q for q in Question.objects.filter(id__in=qids)} + answers = {a.id: a for a in Answer.objects.filter(id__in=aids)} + users = {u.id: u.username for u in User.objects.filter(id__in=pids)} + + for n in notifications: + n.cached_question = questions.get(n.qid) + n.cached_answer = answers.get(n.aid) + n.cached_poster = users.get(n.pid, "Unknown User") + context = { - 'notifications': notifications + 'notifications': notifications, } return render(request, 'website/templates/notifications.html', context) return HttpResponse("go away ...") @@ -836,13 +888,22 @@ def ajax_similar_questions(request): user_title = clean_user_data(title) # Increase the threshold as the Forums questions increase THRESHOLD = 0.3 + MAX_CANDIDATES = 200 + MAX_RESULTS = 20 + top_ques = [] - questions = Question.objects.filter(category=category,tutorial=tutorial) + # Limit number of candidate questions to keep CPU usage bounded + questions = Question.objects.filter( + category=category, + tutorial=tutorial, + ).order_by('-date_created')[:MAX_CANDIDATES] + for question in questions: - question.similarity= get_similar_questions(user_title,question.title) - if question.similarity >= THRESHOLD: + question.similarity = get_similar_questions(user_title, question.title) + if question.similarity >= THRESHOLD: top_ques.append(question) - top_ques = sorted(top_ques,key=lambda x : x.similarity, reverse=True) + + top_ques = sorted(top_ques, key=lambda x: x.similarity, reverse=True)[:MAX_RESULTS] context = { 'questions': top_ques, 'questions_count':len(top_ques) @@ -895,18 +956,23 @@ def ajax_keyword_search(request): questions = Question.objects.filter( Q(title__icontains=key) | Q(category__icontains=key) | Q(tutorial__icontains=key) | Q(body__icontains=key), status=1 - ).order_by('-date_created') + ).annotate(total_answers=Count('answer')).order_by('-date_created') + paginator = Paginator(questions, 20) page = request.POST.get('page') - if page: - page = int(request.POST.get('page')) - questions = paginator.page(page) try: questions = paginator.page(page) except PageNotAnInteger: questions = paginator.page(1) except EmptyPage: questions = paginator.page(paginator.num_pages) + + # Attach cached usernames for the current page + uids = {q.uid for q in questions} + users = {u.id: u.username for u in User.objects.filter(id__in=uids)} + for q in questions: + q.cached_user = users.get(q.uid, "Unknown User") + context = { 'questions': questions } @@ -919,19 +985,38 @@ def ajax_time_search(request): tutorial = request.POST.get('tutorial') minute_range = request.POST.get('minute_range') second_range = request.POST.get('second_range') - questions = None + questions = Question.objects.none() if category: - questions = Question.objects.filter(category=category.replace(' ', '-'), status=1) + questions = Question.objects.filter( + category=category.replace(' ', '-'), + status=1, + ) if tutorial: questions = questions.filter(tutorial=tutorial.replace(' ', '-')) if minute_range: - questions = questions.filter(category=category.replace( - ' ', '-'), tutorial=tutorial.replace(' ', '-'), minute_range=minute_range) + questions = questions.filter(minute_range=minute_range) if second_range: - questions = questions.filter(category=category.replace( - ' ', '-'), tutorial=tutorial.replace(' ', '-'), second_range=second_range) + questions = questions.filter(second_range=second_range) + + questions = questions.annotate(total_answers=Count('answer')).order_by('-date_created') + + paginator = Paginator(questions, 20) + page = request.POST.get('page') + try: + questions = paginator.page(page) + except PageNotAnInteger: + questions = paginator.page(1) + except EmptyPage: + questions = paginator.page(paginator.num_pages) + + # Attach cached usernames for the current page + uids = {q.uid for q in questions} + users = {u.id: u.username for u in User.objects.filter(id__in=uids)} + for q in questions: + q.cached_user = users.get(q.uid, "Unknown User") + context = { - 'questions': questions + 'questions': questions, } return render(request, 'website/templates/ajax-time-search.html', context) @@ -956,27 +1041,30 @@ def forums_mail(to='', subject='', message=''): def unanswered_notification(request): - questions = Question.objects.filter(status=1) + unanswered_questions = ( + Question.objects.filter(status=1) + .annotate(answer_count=Count('answer')) + .filter(answer_count=0) + ) total_count = 0 message = """ The following questions are left unanswered. Please take a look at them.

""" - for question in questions: - if not question.answer_set.count(): - total_count += 1 - message += """ - #{0}
- Title: {1}
- Category: {2}
- Link: {3}
-
- """.format( - total_count, - question.title, - question.category, - 'http://forums.spoken-tutorial.org/question/' + str(question.id) - ) + for question in unanswered_questions: + total_count += 1 + message += """ + #{0}
+ Title: {1}
+ Category: {2}
+ Link: {3}
+
+ """.format( + total_count, + question.title, + question.category, + 'http://forums.spoken-tutorial.org/question/' + str(question.id) + ) to = "team@spoken-tutorial.org, team@fossee.in" subject = "Unanswered questions in the forums." if total_count:
FOSS Tutorial - {{ question.views}} + {{ question.views }} - {{ question.answer_set.count }} + {{ question.total_answers }} - - {{ question.user|truncatechars:10 }} + + {{ question.cached_user|default:question.user|truncatechars:10 }}