Merge branch 'dev' into 'master'

Dev > Master

See merge request dbdev/dbaas!2

Makefile

@@ -228,27 +228,27 @@ docker_stop:
 # a gcp possui mais de um env
 # para acessar os secrets corretos, garantimos que estamos no projeto correto
 set_env:
-	@echo "Project env:${PROJECT_ENV}"; \
-	if [ "${PROJECT_ENV}" = "DEV" ]; then \
+	@echo "Project env:${ENV}"; \
+	if [ "${ENV}" = "DEV" ]; then \
     	echo 'changing project to DEV'; \
     	LOWERCASE_ENV="dev"; \
     	gcloud config set project gglobo-dbaaslab-dev-qa; \
 		exit 0; \
-	elif [ "${PROJECT_ENV}" = "PROD" ]; then \
+	elif [ "${ENV}" = "PROD" ]; then \
 		echo 'changing project to PROD'; \
 		LOWERCASE_ENV="prod"; \
 		gcloud config set project gglobo-dbaas-hub; \
 		exit 0; \
 	else\
-		echo "PROJECT_ENV not found. Exiting";\
-		echo "please call like this: make set_env PROJECT_ENV=PROD or DEV";\
+		echo "ENV not found. Exiting";\
+		echo "please call like this: make set_env ENV=PROD or DEV";\
 		exit 1;\
 	fi\
 
 get_last_tag:
 	@echo "exemplo de uso make get_last_tag ENV=DEV"; \
 	echo "exemplo de uso make get_last_tag ENV=PROD"; \
-	make set_env PROJECT_ENV=${ENV}; \
+	make set_env ENV=${ENV}; \
 	SECRET_NAME="DBDEV_${ENV}_DBAAS_IMAGE_VERSION"; \
 	echo "Getting secret $${SECRET_NAME}"; \
 	gcloud secrets versions access "latest" --secret "$${SECRET_NAME}"; \
@@ -258,7 +258,7 @@ set_new_tag:
 	@echo "tag usada:${TAG}"
 	@echo "exemplo de uso make set_tag TAG=v1.02 ENV=DEV"
 	@echo "exemplo de uso make set_tag TAG=v1.034 ENV=PROD"
-	@make set_env PROJECT_ENV=${ENV}; \
+	@make set_env ENV=${ENV}; \
 	if [ $$? -eq 0 ]; then \
 		echo "env set"; \
 	else \
@@ -282,7 +282,6 @@ docker_deploy_gcp:
 	@echo "make docker_deploy_gcp TAG=v1.02 ENV=PROD"
 	make docker_deploy_build TAG=${TAG}
 	make docker_deploy_push TAG=${TAG}
-	make set_new_tag ENV=${ENV} TAG=${TAG}
 
 docker_deploy_build: 
 	@echo "tag usada:${TAG}"
@@ -344,15 +343,15 @@ gcp_deploy_dev_script:
 
 # Mysql utilities
 gcp_mysql_dev_cli:
-	make set_env PROJECT_ENV=DEV; \
+	make set_env ENV=DEV; \
 	HOST=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_HOST"); \
 	USER=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_USER"); \
 	DB=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_NAME"); \
 	PASS=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_PASSWORD"); \
 	mysql -u$$USER -p$$PASS -h$$HOST
 
 gcp_mysql_dev_dump:
-	make set_env PROJECT_ENV=DEV; \
+	make set_env ENV=DEV; \
 	HOST=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_HOST"); \
 	USER=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_USER"); \
 	DB=$(shell gcloud secrets versions access "latest" --secret "DBDEV_DEV_DBAAS_DBAAS_DATABASE_NAME"); \

dbaas/admin/templates/admin/base.html

@@ -176,9 +176,6 @@
             {% for page in flatpages %}
             <a href="{{ page.url }}">{{ page.title }}</a>&nbsp;&nbsp;&nbsp;
             {% endfor %}
-            <a href="{{ iaas_status }}" target="_blank">IaaS status</a>&nbsp;&nbsp;&nbsp;
-            <a href="{{ iaas_quota }}" target="_blank">IaaS quota</a>&nbsp;&nbsp;&nbsp;
-            <a href="{{ sofia_main_dashboard }}" target="_blank">Sofia Dashboard</a>&nbsp;&nbsp;&nbsp;
             </p>
         </div>
     {% endblock %}

dbaas/admin/templatetags/config_tags.py

@@ -19,3 +19,12 @@ def get_config(conf_name=None):
         "description": c.description,
         "hash": c.hash
     }
+
+
+@register.filter
+def is_dba(user, name):
+    list_validation = Configuration.get_by_name_as_list('list_validation_custom_views')
+    is_dba = user.team_set.filter(role__name="role_dba").exists()
+    if is_dba or name in list_validation:
+        return True
+    return False

dbaas/backup/tasks.py

@@ -67,7 +67,7 @@ def mysql_binlog_save(client, instance):
 def lock_instance(driver, instance, client):
     try:
         LOG.debug('Locking instance {}'.format(instance))
-        driver.lock_database(client)
+        driver.lock_database_ssh(instance, client)
         LOG.debug('Instance {} is locked'.format(instance))
         return True
     except Exception as e:
@@ -78,7 +78,7 @@ def lock_instance(driver, instance, client):
 def unlock_instance(driver, instance, client):
     try:
         LOG.debug('Unlocking instance {}'.format(instance))
-        driver.unlock_database(client)
+        driver.unlock_database_ssh(instance, client)
         LOG.debug('Instance {} is unlocked'.format(instance))
         return True
     except Exception as e:
@@ -230,70 +230,78 @@ def make_instance_gcp_snapshot_backup(
 
     snapshot_final_status = Snapshot.SUCCESS
 
-    locked = None
-    client = None
     driver = infra.get_driver()
+    client = driver.get_client(instance)
     try:
-        client = driver.get_client(instance)
         locked = lock_instance(driver, instance, client)
         if not locked:
             snapshot_final_status = Snapshot.WARNING
 
         if 'MySQL' in type(driver).__name__:
             mysql_binlog_save(client, instance)
 
-        has_snapshot = Snapshot.objects.filter(
+        has_snapshot_with_status_warning = Snapshot.objects.filter(
             status=Snapshot.WARNING, instance=instance, end_at__year=datetime.now().year,
             end_at__month=datetime.now().month, end_at__day=datetime.now().day
         )
-        backup_hour_list = Configuration.get_by_name_as_list('make_database_backup_hour')
-        if not snapshot_final_status == Snapshot.WARNING and not has_snapshot:
-            cont = 0
-            for _ in range(backup_retry_attempts):
-                cont += 1
-                try:
-                    code = 201
-                    response, data = provider.new_take_snapshot(persist=persist)
 
-                    if response.status_code < 400:
-                        break
+        if has_snapshot_with_status_warning:
+            raise Exception("Backup with WARNING already created today.")
 
-                    if cont >= 3:
-                        raise IndexError
+        if snapshot_final_status == Snapshot.WARNING:
+            raise Exception("Snapshot has status WARNING, check the logs.")
 
-                except IndexError as e:
-                    response, content = e
-                    if response.status_code == 503:
-                        errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
-                            strftime("%d/%m/%Y %H:%M:%S"), instance
-                        )
-                        LOG.error(errormsg)
-                        if task:
-                            task.add_detail(errormsg)
-                        sleep(30)
-                    else:
-                        raise e
+        cont = 0
+        for _ in range(backup_retry_attempts):
+            cont += 1
+            try:
+                code = 201
+                response, data = provider.new_take_snapshot(persist=persist)
 
-            if response.status_code < 400:
-                while code != 200:
-                    sleep(20)
-                    snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
-                    if snap_response.status_code in [200, 202]:
-                        unlock_instance(driver, instance, client)
-                    if snap_response.status_code == 200:
-                        break
-                    if snap_response.status_code >= 400:
-                        raise error
-                    code = snap_response.status_code
-
-                snapshot.done(snap_status)
-                snapshot.save()
-            else:
-                errormsg = response['message']
-                set_backup_error(infra, snapshot, errormsg)
+                if response.status_code < 400:
+                    break
+
+                if cont >= 3:
+                    raise IndexError
+
+            except IndexError as e:
+                response, content = e
+                if response.status_code == 503:
+                    errormsg = "{} - 503 error creating snapshot for instance: {}. It will try again in 30 seconds. ".format(
+                        strftime("%d/%m/%Y %H:%M:%S"), instance
+                    )
+                    LOG.error(errormsg)
+                    if task:
+                        task.add_detail(errormsg)
+                    sleep(30)
+                else:
+                    raise e
+
+        count = 0
+        if response.status_code < 400:
+            while code != 200:
+                sleep(20)
+                snap_response, snap_status = provider.take_snapshot_status(data['identifier'])
+                if snap_response.status_code in [200, 202]:
+                    unlock_instance(driver, instance, client)
+                if snap_response.status_code == 200:
+                    break
+                if snap_response.status_code >= 400:
+                    raise error
+                if count >= 90:
+                    errormsg = "Timeout"
+                    set_backup_error(infra, snapshot, errormsg)
+                    raise Exception(errormsg)
+                else:
+                    count += 1
+                code = snap_response.status_code
+
+            snapshot.done(snap_status)
+            snapshot.save()
         else:
-            if str(current_hour) in backup_hour_list:
-                raise Exception("Backup with WARNING already created today.")
+            errormsg = response['message']
+            set_backup_error(infra, snapshot, errormsg)
+            raise Exception(errormsg)
 
     except Exception as e:
         errormsg = "Error creating snapshot: {}".format(e)

dbaas/dashboard/urls.py

@@ -5,8 +5,6 @@
 
                        url(r"^$", "dashboard", name="dashboard.index"),
 
-                       url(r'^sofia_dashboard/$', "sofia_dashboard"),
-
                        url(r'^databaseinfra/(?P<infra_id>\d+)/?$',
                            'databaseinfra', name='databaseinfra.index'),
 

dbaas/dbaas/settings.py

@@ -252,7 +252,6 @@ def LOCAL_FILES(path):
     "django.core.context_processors.tz",
     "django.contrib.messages.context_processors.messages",
     'django.core.context_processors.request',
-    'dbaas.views.external_links',
 )
 
 # backend for haystack

dbaas/dbaas/templates/adminplus/index.html

@@ -1,4 +1,5 @@
 {% extends "admin/index.html" %}
+{% load config_tags %}
 
 {% block sidebar %}
   {{ block.super }}
@@ -20,7 +21,9 @@
                 </thead>
                 <tbody>
                     {% for path, name in custom_list %}
-                    <tr><td><a href="{{ path }}">{{ name }}</a></td></tr>
+                    {% if user|is_dba:name %}
+                        <tr><td><a href="{{ path }}">{{ name }}</a></td></tr>
+                        {% endif %}
                     {% endfor %}
                 </tbody>
             </table>

dbaas/dbaas/views.py

@@ -1,18 +1,5 @@
-from system.models import Configuration
 from django.views.generic import TemplateView
 
 
-def external_links(request):
-    iaas_status = Configuration.get_by_name('iaas_status')
-    iaas_quota = Configuration.get_by_name('iaas_quota')
-    sofia_grafana_url = Configuration.get_by_name('sofia_grafana_url')
-    sofia_grafana_datasource = Configuration.get_by_name('sofia_grafana_datasource')
-    sofia_dashboard = "{}?var-datasource={}".format(sofia_grafana_url, sofia_grafana_datasource)
-
-    return {'iaas_status': iaas_status,
-            'iaas_quota': iaas_quota,
-            'sofia_main_dashboard': sofia_dashboard}
-
-
 class DeployView(TemplateView):
     template_name = 'deploy/deploy.html'

dbaas/dbaas_services/analyzing/views.py

@@ -42,28 +42,30 @@ def post(self, request, *args, **kwargs):
 
 class DatabaseReport(ListView):
 
-    def has_perm(self, request):
+    def has_permission(self, request):
         UserMiddleware.set_current_user(request.user)
         from_teams = [x.role for x in Team.objects.filter(users=request.user)]
         role_dba = Role.objects.get(name='role_dba')
 
-        if role_dba not in from_teams:
-            return render(request, 'databases/index.html', {"has_perm": False})
+        if role_dba in from_teams:
+            return render(request, 'databases/index.html', {"has_perm": True})
+
 
     def get(self, request, *args, **kwargs):
-        self.has_perm(request)
+        if self.has_permission(request):
+            return render(request, 'databases/index.html', {"has_perm": True})
+        else:
+            return HttpResponseRedirect(reverse('admin:index'))
 
-        return render(request, 'databases/index.html', {"has_perm": True})
 
     def post(self, request, *args, **kwargs):
-        self.has_perm(request)
+        if self.has_permission(request):
+            database_report = request.POST.get("database_report", "")
+            if database_report == 'database_report':
+                return self.default_database_report()
+            else:
+                return self.vm_by_line_database_report()
 
-        database_report = request.POST.get("database_report", "")
-
-        if database_report == 'database_report':
-            return self.default_database_report()
-        else:
-            return self.vm_by_line_database_report()
 
     def vm_by_line_database_report(self):
 

dbaas/drivers/base.py

@@ -214,9 +214,15 @@ def get_client(self, instance):
     def lock_database(self, client):
         raise NotImplementedError()
 
+    def lock_database_ssh(self, instance, client):
+        return self.lock_database(client)
+
     def unlock_database(self, client):
         raise NotImplementedError()
 
+    def unlock_database_ssh(self, instance, client):
+        return self.unlock_database(client)
+
     def check_instance_is_eligible_for_backup(self, instance):
         raise NotImplementedError()