Conclusion of 300 thousand records in Django with pagination

I need to display hundreds of thousands of records in the form of a table with pagination. I use Django 2.0.3.

view.py

from django.shortcuts import render_to_response from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage from django.views import View from parsercore.models import LogEntry as Entries class IndexView(View): template_name = 'parsercore/search.html' #form_class = UrlForm def get(self, request, *args, **kwargs): context = {} return render_to_response(self.template_name, context=context) class ResultView(View): template_name = 'parsercore/result-page.html' context_object_name = 'search_entries' def get_queryset(self): """Return the last log entries.""" entry_count = Entries.objects.count() print('количество записей в выборке ', entry_count) return Entries.objects.order_by('timestamp') def get(self, request, *args, **kwargs): context = {} search_entries = self.get_queryset() paginator = Paginator(search_entries, 50) page = request.GET.get('page') print('page: ', page) if page is None: page = 1 results = paginator.get_page(page) context['results'] = results context['result_lists'] = search_entries return render_to_response(template_name='parsercore/result-page.html', context=context) 

models.py

 import os from django.db import models from django.utils.translation import ugettext as _ import datetime class LogFilePath(models.Model): """ Allows for multiple logs to be stored in the same path, saving on database usage and allowing for easy migration of the database from one server to another. """ path = models.TextField( unique=False, help_text=_("The base path for one or more log files."), ) def __unicode__(self): return self.path class LogFile(models.Model): """ Representation of a log file. """ path = models.ForeignKey( LogFilePath, help_text=_("The base path for this log file."), on_delete=models.CASCADE, ) filename = models.CharField( max_length=200, help_text=_("The base file name of the log file within the base path."), ) created = models.DateTimeField( auto_now_add=True, help_text=_("When this log file was created in the database."), ) modified = models.DateTimeField( auto_now=True, help_text=_("The last date/time that this log file was modified."), ) parsed = models.DateTimeField( blank=True, null=True, help_text=_("The date/time at which this log file was completely parsed."), ) def __unicode__(self): return os.path.join(u'%s' % self.path, u'%s' % self.filename) class LogEntry(models.Model): """ A single web log entry from a specific log file. """ log_file = models.ForeignKey( LogFile, help_text=_("The log file in which this entry was found."), db_index=True, related_name='entries', on_delete=models.CASCADE, ) remote_host = models.CharField( max_length=100, help_text=_("The remote host which made the request (%%h)."), ) client_id = models.CharField( max_length=100, help_text=_("RFC 1413 identity of the client (%%l)."), ) user_id = models.CharField( max_length=100, help_text=_("The user ID of the user requesting the document (%%u)."), ) timestamp = models.DateTimeField( help_text=_("The date/time that the request was received (%%t)."), db_index=True, blank=True, null=True, default=datetime.date.today ) method = models.TextField( help_text=_("The method of request line from the client (%%r)."), ) request = models.TextField( help_text=_("The request line from the client (%%r)."), ) status = models.CharField( #Integer max_length=100, help_text=_("The status code that the server sent back to the client (%%>s)."), db_index=True, ) bytes_returned = models.CharField( #Integer max_length=100, help_text=_("The size of the object returned to the client (%%b)."), ) referer = models.TextField( blank=True, null=True, help_text=_("The \"Referer\" HTTP request header (\\\"%%{Referer}i\\\")."), ) user_agent = models.TextField( blank=True, null=True, help_text=_("The \"User-Agent\" HTTP request header (\\\"%%{User-Agent}i\\\")."), db_index=True, ) session_id = models.CharField( max_length=100, blank=True, null=True, help_text=_("The session ID, if any, depending on whether this is configured."), db_index=True, ) # metadata parameters built up from entries path = models.TextField( blank=True, null=True, help_text=_("The path requested from the web server."), ) is_robot = models.BooleanField( default=False, help_text=_("Was this request generated by some sort of robot?"), ) is_page = models.BooleanField( default=True, help_text=_("Was the response most likely an HTML document?"), ) class Meta: verbose_name = _('log entry') verbose_name_plural = _('log entries') def __unicode__(self): return _("Request at %(timestamp)s from %(remote)s") % {'timestamp': self.timestamp.strftime("%Y-%m-%d %H:%M:%S %z"), 'remote': self.remote_host} 

urls.py

 from django.contrib import admin from django.urls import path from django.conf.urls import url from . import views app_name = 'parsercore' urlpatterns = [ url(r'^$', views.IndexView.as_view(), name='index'), url(r'^show/', views.ResultView.as_view(), name='show'), ] 

and

 from django.contrib import admin from django.urls import path, include from parsercore import views urlpatterns = [ path(r'search/', include('parsercore.urls', namespace='search')), path(r'admin/', admin.site.urls), path(r'', include('parsercore.urls', namespace='search')), ] 

result-page.html

 {% extends 'parsercore/base.html' %} {% load bootstrap3 %} {% block content %} <nav class="navbar navbar-dark bg-dark justify-content-between"> <a class="navbar-brand"></a> <form class="form-inline"> <input class="form-control mr-sm-2" type="search" placeholder="Поиск по результатам" aria-label="Search"> <button class="btn btn-outline-success my-2 my-sm-0" type="submit">Search</button> </form> </nav> <h1>Результаты</h1> {% if result_lists %} <table class="table table-striped"> <thead class="thead-dark"><tr> <th scope="col">IP</th> <th scope="col">Дата</th> <th scope="col">Метод</th> <th scope="col">Запрос</th> <th scope="col">Код</th> <th scope="col">Размер ответа</th> </tr></thead><tbody> {% for result in result_lists %} <tr> <td>{{ result.remote_host }}</td> <td>{{ result.timestamp }}</td> <td>{{ result.method }}</td> <td>{{ result.request }}</td> <td>{{ result.status }}</td> <td>{{ result.bytes_returned }}</td> </tr> {% endfor %} </tbody></table> <div class="pagination"> <span class="step-links"> {% if results.has_previous %} <a href="?page=1">&laquo; first</a> <a href="?page={{ results.previous_page_number }}">previous</a> {% endif %} <span class="current"> Page {{ results.number }} of {{ results.paginator.num_pages }}. </span> {% if results.has_next %} <a href="?page={{ results.next_page_number }}">next</a> <a href="?page={{ results.paginator.num_pages }}">last &raquo;</a> {% endif %} </span> </div> {% else %} <p>Не найдено записей в бд по вашему запросу<br>Попробуйте загрузить файл еще раз</p> {% endif %} {% endblock %} 

The page normally works up to a thousand entries. When eliminating the number limit, the page is not loaded, processing takes place within a minute, then the message MemoryError appears in the console. I also need to make statistics (maybe using filtering) for the same query.

UPD: the solution is found, look in the comments.