Merge pull request #19 from patrickpr/v1.1

version 1.0.2 merge

.travis.yml

@@ -14,8 +14,8 @@ before_script:
   - ./tests/install_dep.sh
 
 env:
-  - DB=mysql DBVER=1
-  - DB=pgsql DBVER=1
+  - DB=mysql DBVER=2
+  - DB=pgsql DBVER=2
 
 script:
   - ./tests/launch_test.sh

Changelog.txt

@@ -1,10 +1,12 @@
 -------------------------------------------------------------------------------------------------
 Version 1.0.2
-Date : No released
-Database update : No
+Date : 04 Nov 2019
+Database update : Yes
 Migration path : from version > 1.0.0
 
 UPGRADE NOTES : 
+The main changes are the way mib updates are handled.
+First update is now twice as fast, and incremental updates don't re-create the database.
 
 BUGS
 ====
@@ -13,8 +15,9 @@ BUGS
 
 ENHANCEMENTS
 ============
-
-1) Added regexp to select oid for rule eval (not implemented in GUI) 
+1) Added regexp to select oid for rule eval (not implemented in GUI)
+2) Mib database update is now much faster.
+3) Automatic bash script installer (see documentation).
 
 -------------------------------------------------------------------------------------------------
 Version 1.0.1

README.md

@@ -12,7 +12,7 @@ Projet features :
 - Update icinga services based on rules : host or hostgroups and traps data updates service status.
 - OID decode to human readable name, possible to add mib files via web.
 
-Project status : Stable release 1.0.1
+Project status : Stable release 1.0.2
 
 - Module has been installed and tested on CentOS 7, Ubuntu 18.04 (Bionic) and some more.
 - All project feature are working on those systems.	 

SQL/schema_v2.pgsql

@@ -0,0 +1,98 @@
+CREATE TABLE #PREFIX#db_config (
+	id serial NOT NULL,
+    name character varying(100) NOT NULL,
+    value text,
+    PRIMARY KEY (id)
+);
+
+INSERT INTO #PREFIX#db_config (name,value) VALUES ('db_version',2);
+
+CREATE OR REPLACE FUNCTION unix_timestamp(timestamp with time zone) RETURNS bigint AS '
+        SELECT EXTRACT(EPOCH FROM $1)::bigint AS result
+' LANGUAGE sql;
+
+CREATE TABLE #PREFIX#mib_cache (
+  id serial NOT NULL,
+  oid character varying(256) NOT NULL,
+  mib character varying(256) NOT NULL,
+  name character varying(512) NOT NULL,
+  type character varying(256) DEFAULT NULL,
+  textual_convention character varying(256) DEFAULT NULL,
+  display_hint character varying(256) DEFAULT NULL,
+  syntax character varying(256) DEFAULT NULL,
+  type_enum text,
+  description text,
+  PRIMARY KEY (id)
+) ;
+
+CREATE TABLE #PREFIX#mib_cache_trap_object (
+  id serial NOT NULL,
+  trap_id bigint NOT NULL,
+  object_id bigint NOT NULL,
+  PRIMARY KEY (id),
+  CONSTRAINT fk_object_id_obj FOREIGN KEY (object_id)
+    REFERENCES #PREFIX#mib_cache (id)
+    ON UPDATE CASCADE
+    ON DELETE CASCADE,  
+  CONSTRAINT FK_trap_id_obj FOREIGN KEY (trap_id) 
+	REFERENCES #PREFIX#mib_cache (id) 
+	ON DELETE CASCADE 
+	ON UPDATE CASCADE
+) ;
+
+CREATE TYPE trapstate AS ENUM
+    ('done', 'waiting', 'unknown', 'error');
+
+CREATE TABLE #PREFIX#received (
+  id serial NOT NULL,
+  source_ip character varying(45) DEFAULT NULL,
+  source_port integer  DEFAULT NULL,
+  destination_ip character varying(45) DEFAULT NULL,
+  destination_port integer DEFAULT NULL,
+  trap_oid character varying(256) DEFAULT NULL,
+  date_received TIMESTAMPTZ NOT NULL,
+  status trapstate NOT NULL DEFAULT 'waiting',
+  trap_name character varying(256) DEFAULT NULL,
+  source_name character varying(256) DEFAULT NULL,
+  trap_name_mib character varying(100) DEFAULT NULL,
+  process_time float DEFAULT '0',
+  status_detail character varying(256) DEFAULT NULL,
+  PRIMARY KEY (id)
+) ;
+
+CREATE TABLE #PREFIX#received_data (
+  id serial NOT NULL,
+  oid character varying(256) DEFAULT NULL,
+  value character varying(1024) DEFAULT NULL,
+  trap_id bigint  NOT NULL,
+  oid_name character varying(256) DEFAULT NULL,
+  oid_name_mib character varying(100) DEFAULT NULL,
+  PRIMARY KEY (id),
+  CONSTRAINT FK_trap_id FOREIGN KEY (trap_id) 
+	REFERENCES #PREFIX#received (id) 
+	ON DELETE CASCADE
+) ;
+
+CREATE TABLE #PREFIX#rules (
+  id serial NOT NULL,
+  ip4 character varying(20) DEFAULT NULL,
+  ip6 character varying(42) DEFAULT NULL,
+  trap_oid character varying(256) NOT NULL,
+  host_name character varying(256) DEFAULT NULL,
+  host_group_name character varying(256) DEFAULT NULL,
+  rule text,
+  action_match smallint NOT NULL DEFAULT '-1',
+  action_nomatch smallint NOT NULL DEFAULT '-1',
+  service_name character varying(256) NOT NULL,
+  revert_ok bigint NOT NULL DEFAULT '3600',
+  display_nok text,
+  display text,
+  created TIMESTAMPTZ DEFAULT NULL,
+  modified TIMESTAMPTZ DEFAULT NULL,
+  modifier character varying(100) DEFAULT NULL,
+  num_match bigint DEFAULT '0',
+  num_match_nok bigint DEFAULT NULL,
+  comment text,
+  rule_type integer DEFAULT NULL,
+  PRIMARY KEY (id)
+) ;

SQL/schema_v2.sql

@@ -0,0 +1,87 @@
+CREATE TABLE `#PREFIX#db_config` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(100) NOT NULL,
+  `value` mediumtext,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+INSERT INTO #PREFIX#db_config (`name`,`value`) VALUES ('db_version',2);
+
+CREATE TABLE #PREFIX#mib_cache (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  oid varchar(256) NOT NULL,
+  mib varchar(256) NOT NULL,
+  name varchar(512) NOT NULL,
+  type varchar(256) DEFAULT NULL,
+  textual_convention varchar(256) DEFAULT NULL,
+  display_hint varchar(256) DEFAULT NULL,
+  syntax varchar(256) DEFAULT NULL,
+  type_enum text,
+  description text,
+  PRIMARY KEY (id)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+CREATE TABLE #PREFIX#mib_cache_trap_object (
+  id int(12) NOT NULL AUTO_INCREMENT,
+  trap_id int(11) NOT NULL,
+  object_id int(11) NOT NULL,
+  PRIMARY KEY (id),
+  KEY FK_trap_id_obj_idx (trap_id),
+  KEY FK_object_id_obj_idx (object_id),
+  CONSTRAINT FK_object_id_obj FOREIGN KEY (object_id) REFERENCES #PREFIX#mib_cache (id) ON DELETE CASCADE ON UPDATE CASCADE,
+  CONSTRAINT FK_trap_id_obj FOREIGN KEY (trap_id) REFERENCES #PREFIX#mib_cache (id) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+CREATE TABLE `#PREFIX#received` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `source_ip` varchar(45) DEFAULT NULL,
+  `source_port` smallint(5) unsigned DEFAULT NULL,
+  `destination_ip` varchar(45) DEFAULT NULL,
+  `destination_port` smallint(5) unsigned DEFAULT NULL,
+  `trap_oid` varchar(256) DEFAULT NULL,
+  `date_received` datetime NOT NULL,
+  `status` enum('done','waiting','unknown','error') NOT NULL DEFAULT 'waiting',
+  `trap_name` varchar(256) DEFAULT NULL,
+  `source_name` varchar(256) DEFAULT NULL,
+  `trap_name_mib` varchar(100) DEFAULT NULL,
+  `process_time` float DEFAULT '0',
+  `status_detail` varchar(256) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+CREATE TABLE `#PREFIX#received_data` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `oid` varchar(256) DEFAULT NULL,
+  `value` varchar(1024) DEFAULT NULL,
+  `trap_id` int(11) unsigned NOT NULL,
+  `oid_name` varchar(256) DEFAULT NULL,
+  `oid_name_mib` varchar(100) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `FK_trap_id_idx` (`trap_id`),
+  CONSTRAINT `FK_trap_id` FOREIGN KEY (`trap_id`) REFERENCES `#PREFIX#received` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+CREATE TABLE `#PREFIX#rules` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `ip4` varchar(20) DEFAULT NULL,
+  `ip6` varchar(42) DEFAULT NULL,
+  `trap_oid` varchar(256) NOT NULL,
+  `host_name` varchar(256) DEFAULT NULL,
+  `host_group_name` varchar(256) DEFAULT NULL,
+  `rule` text,
+  `action_match` tinyint(5) NOT NULL DEFAULT '-1',
+  `action_nomatch` tinyint(5) NOT NULL DEFAULT '-1',
+  `service_name` varchar(256) NOT NULL,
+  `revert_ok` int(11) NOT NULL DEFAULT '3600',
+  `display_nok` text,
+  `display` text,
+  `created` datetime DEFAULT NULL,
+  `modified` datetime DEFAULT NULL,
+  `modifier` varchar(100) DEFAULT NULL,
+  `num_match` int(16) DEFAULT '0',
+  `num_match_nok` int(16) DEFAULT NULL,
+  `comment` text,
+  `rule_type` int(8) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+

SQL/update_pgsql/schema_v1_v2.sql

@@ -0,0 +1,38 @@
+#MESSAGE : This upgrade will drop all mib cache table. You will have to do a mib database update in status&mibs -> MIB Management -> Update.
+#PRE-SCRIPT :
+#POST-SCRIPT :
+
+DROP TABLE #PREFIX#mib_cache_syntax;
+DROP TABLE #PREFIX#mib_cache_tc;
+DROP TABLE #PREFIX#mib_cache_trap_object;
+DROP TABLE #PREFIX#mib_cache;
+
+CREATE TABLE #PREFIX#mib_cache (
+  id serial NOT NULL,
+  oid character varying(256) NOT NULL,
+  mib character varying(256) NOT NULL,
+  name character varying(512) NOT NULL,
+  type character varying(256) DEFAULT NULL,
+  textual_convention character varying(256) DEFAULT NULL,
+  display_hint character varying(256) DEFAULT NULL,
+  syntax character varying(256) DEFAULT NULL,
+  type_enum text,
+  description text,
+  PRIMARY KEY (id)
+) ;
+
+CREATE TABLE #PREFIX#mib_cache_trap_object (
+  id serial NOT NULL,
+  trap_id bigint NOT NULL,
+  object_id bigint NOT NULL,
+  PRIMARY KEY (id),
+  CONSTRAINT fk_object_id_obj FOREIGN KEY (object_id)
+    REFERENCES #PREFIX#mib_cache (id)
+    ON UPDATE CASCADE
+    ON DELETE CASCADE,  
+  CONSTRAINT FK_trap_id_obj FOREIGN KEY (trap_id) 
+	REFERENCES #PREFIX#mib_cache (id) 
+	ON DELETE CASCADE 
+	ON UPDATE CASCADE
+) ;
+

SQL/update_sql/schema_v1_v2.sql

@@ -0,0 +1,34 @@
+#MESSAGE : This upgrade will drop all mib cache table. You will have to do a mib database update in status&mibs -> MIB Management -> Update.
+#PRE-SCRIPT :
+#POST-SCRIPT :
+
+DROP TABLE #PREFIX#mib_cache_syntax;
+DROP TABLE #PREFIX#mib_cache_tc;
+DROP TABLE #PREFIX#mib_cache_trap_object;
+DROP TABLE #PREFIX#mib_cache;
+
+CREATE TABLE #PREFIX#mib_cache (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  oid varchar(256) NOT NULL,
+  mib varchar(256) NOT NULL,
+  name varchar(512) NOT NULL,
+  type varchar(256) DEFAULT NULL,
+  textual_convention varchar(256) DEFAULT NULL,
+  display_hint varchar(256) DEFAULT NULL,
+  syntax varchar(256) DEFAULT NULL,
+  type_enum text,
+  description text,
+  PRIMARY KEY (id)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+CREATE TABLE #PREFIX#mib_cache_trap_object (
+  id int(12) NOT NULL AUTO_INCREMENT,
+  trap_id int(11) NOT NULL,
+  object_id int(11) NOT NULL,
+  PRIMARY KEY (id),
+  KEY FK_trap_id_obj_idx (trap_id),
+  KEY FK_object_id_obj_idx (object_id),
+  CONSTRAINT FK_object_id_obj FOREIGN KEY (object_id) REFERENCES #PREFIX#mib_cache (id) ON DELETE CASCADE ON UPDATE CASCADE,
+  CONSTRAINT FK_trap_id_obj FOREIGN KEY (trap_id) REFERENCES #PREFIX#mib_cache (id) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+

application/clicommands/MibCommand.php

@@ -29,13 +29,17 @@ class MibCommand extends Command
 	*	OPTIONS
 	*	
 	*	--pid <file> : run in background with pid in <file>
+	*
 	*	--verb    : Set output log to verbose
 	*
+	*   --force-check : force check of all traps & objects for change. (NOT IMPLEMENTED)
 	*/
 	public function updateAction()
 	{
 	    $background = $this->params->get('pid', null);
 	    $logLevel= $this->params->has('verb') ? 4 : 2;
+	    if ($this->params->has('force-check')) { echo "Not implemented"; return;}
+	    $forceCheck=$this->params->has('force-check')?True:False;
 	    $pid=1;
 	    if ($background != null)
 	    {
@@ -78,8 +82,7 @@ public function updateAction()
                 fclose(STDERR);
                 try
                 {
-                    $trap->update_mib_database(false);
-                    $trap->update_mibs_options();
+                    $trap->update_mib_database(false,$forceCheck);
                 }
                 catch (Exception $e)
                 {
@@ -94,10 +97,9 @@ public function updateAction()
 		try
 		{
 			echo "Update main mib database : \n";
-			$trap->update_mib_database(true);
-			echo "Updating options : \n";
-			$trap->update_mibs_options();
-			echo "Done : \n";
+			echo "# (trap found) C (trap already processed) . (every 2 seconds) : \n";
+			$trap->update_mib_database(true,$forceCheck);
+			echo "Done\n";
 
 		}
 		catch (Exception $e)

application/controllers/HelperController.php

@@ -243,7 +243,8 @@ public function translateoidAction()
 					'mib' => $object['mib'], 
 					'name' => $object['name'],
 					'type' => $object['type'],
-					'type_enum' => $object['type_enum']
+					'type_enum' => $object['type_enum'],
+				    'description' => $object['description']
 				)
 			);
 		}