file storage for aqo_query_texts

.gitignore

@@ -8,3 +8,6 @@ regression.out
 *.gcov
 tags
 
+# Generated subdirectories
+/log/
+/tmp_check/

Makefile

@@ -6,7 +6,7 @@ PGFILEDESC = "AQO - Adaptive Query Optimization"
 MODULE_big = aqo
 OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
 hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
-selectivity_cache.o storage.o utils.o ignorance.o new_storage.o $(WIN32RES)
+selectivity_cache.o storage.o utils.o ignorance.o file_storage.o $(WIN32RES)
 
 TAP_TESTS = 1
 
@@ -22,6 +22,7 @@ REGRESS =	aqo_disabled \
 			forced_stat_collection \
 			unsupported \
 			clean_aqo_data \
+			plancache	\
 			top_queries
 
 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw

aqo--1.0.sql

@@ -2,20 +2,20 @@
 \echo Use "CREATE EXTENSION aqo" to load this file. \quit
 
 CREATE TABLE public.aqo_queries (
-	query_hash		int PRIMARY KEY,
+	query_hash		bigint PRIMARY KEY,
 	learn_aqo		boolean NOT NULL,
 	use_aqo			boolean NOT NULL,
-	fspace_hash		int NOT NULL,
+	fspace_hash		bigint NOT NULL,
 	auto_tuning		boolean NOT NULL
 );
 
 CREATE TABLE public.aqo_query_texts (
-	query_hash		int PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
+	query_hash		bigint PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
 	query_text		varchar NOT NULL
 );
 
 CREATE TABLE public.aqo_query_stat (
-	query_hash		int PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
+	query_hash		bigint PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
 	execution_time_with_aqo					double precision[],
 	execution_time_without_aqo				double precision[],
 	planning_time_with_aqo					double precision[],
@@ -27,7 +27,7 @@ CREATE TABLE public.aqo_query_stat (
 );
 
 CREATE TABLE public.aqo_data (
-	fspace_hash		int NOT NULL REFERENCES public.aqo_queries ON DELETE CASCADE,
+	fspace_hash		bigint NOT NULL REFERENCES public.aqo_queries ON DELETE CASCADE,
 	fsspace_hash	int NOT NULL,
 	nfeatures		int NOT NULL,
 	features		double precision[][],

aqo--1.1--1.2.sql

@@ -28,12 +28,12 @@ DROP FUNCTION aqo_migrate_to_1_2_get_pk(regclass);
 --
 
 -- Show query state at the AQO knowledge base
-CREATE OR REPLACE FUNCTION public.aqo_status(hash int)
+CREATE OR REPLACE FUNCTION public.aqo_status(hash bigint)
 RETURNS TABLE (
 	"learn"			BOOL,
 	"use aqo"		BOOL,
 	"auto tune"		BOOL,
-	"fspace hash"	INT,
+	"fspace hash"	bigINT,
 	"t_naqo"		TEXT,
 	"err_naqo"		TEXT,
 	"iters"			BIGINT,
@@ -63,7 +63,7 @@ WHERE (aqs.query_hash = aq.query_hash) AND
 	aqs.query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE OR REPLACE FUNCTION public.aqo_enable_query(hash int)
+CREATE OR REPLACE FUNCTION public.aqo_enable_query(hash bigint)
 RETURNS VOID
 AS $func$
 UPDATE public.aqo_queries SET
@@ -72,7 +72,7 @@ UPDATE public.aqo_queries SET
 	WHERE query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE OR REPLACE FUNCTION public.aqo_disable_query(hash int)
+CREATE OR REPLACE FUNCTION public.aqo_disable_query(hash bigint)
 RETURNS VOID
 AS $func$
 UPDATE public.aqo_queries SET
@@ -82,7 +82,7 @@ UPDATE public.aqo_queries SET
 	WHERE query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE OR REPLACE FUNCTION public.aqo_clear_hist(hash int)
+CREATE OR REPLACE FUNCTION public.aqo_clear_hist(hash bigint)
 RETURNS VOID
 AS $func$
 DELETE FROM public.aqo_data WHERE fspace_hash=$1;
@@ -96,7 +96,7 @@ SELECT query_hash FROM public.aqo_query_stat aqs
 	WHERE -1 = ANY (cardinality_error_with_aqo::double precision[]);
 $func$ LANGUAGE SQL;
 
-CREATE OR REPLACE FUNCTION public.aqo_drop(hash int)
+CREATE OR REPLACE FUNCTION public.aqo_drop(hash bigint)
 RETURNS VOID
 AS $func$
 DELETE FROM public.aqo_queries aq WHERE (aq.query_hash = $1);

aqo--1.2--1.3.sql

@@ -10,7 +10,7 @@ DECLARE
     aqo_query_texts_row aqo_query_texts%ROWTYPE;
     aqo_query_stat_row aqo_query_stat%ROWTYPE;
     oid_var oid;
-    fspace_hash_var int;
+    fspace_hash_var bigint;
     delete_row boolean DEFAULT false;
 BEGIN
   RAISE NOTICE 'Cleaning aqo_data records';
@@ -89,8 +89,8 @@ $$ LANGUAGE plpgsql;
 --
 CREATE OR REPLACE FUNCTION public.top_time_queries(n int)
   RETURNS TABLE(num bigint,
-                fspace_hash int,
-                query_hash int,
+                fspace_hash bigint,
+                query_hash bigint,
                 execution_time float,
                 deviation float
                )
@@ -115,8 +115,8 @@ $$ LANGUAGE plpgsql;
 --
 CREATE OR REPLACE FUNCTION public.top_error_queries(n int)
   RETURNS TABLE(num bigint,
-                fspace_hash int,
-                query_hash int,
+                fspace_hash bigint,
+                query_hash bigint,
                 error float,
                 deviation float
                )
@@ -135,3 +135,14 @@ BEGIN
     ORDER BY error DESC LIMIT n;
 END;
 $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION public.file_read_query_text(
+    OUT query_hash integer, 
+    OUT query_text text
+)
+RETURNS setof record
+AS 'MODULE_PATHNAME', 'file_read_query_text'
+LANGUAGE C VOLATILE;
+
+CREATE VIEW file_read_query_texts AS
+	SELECT * FROM public.file_read_query_text();

aqo--1.2.sql

@@ -2,20 +2,20 @@
 \echo Use "CREATE EXTENSION aqo" to load this file. \quit
 
 CREATE TABLE public.aqo_queries (
-	query_hash		int CONSTRAINT aqo_queries_query_hash_idx PRIMARY KEY,
+	query_hash		bigint CONSTRAINT aqo_queries_query_hash_idx PRIMARY KEY,
 	learn_aqo		boolean NOT NULL,
 	use_aqo			boolean NOT NULL,
-	fspace_hash		int NOT NULL,
+	fspace_hash		bigint NOT NULL,
 	auto_tuning		boolean NOT NULL
 );
 
 CREATE TABLE public.aqo_query_texts (
-	query_hash		int CONSTRAINT aqo_query_texts_query_hash_idx PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
+	query_hash		bigint CONSTRAINT aqo_query_texts_query_hash_idx PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
 	query_text		text NOT NULL
 );
 
 CREATE TABLE public.aqo_query_stat (
-	query_hash		int CONSTRAINT aqo_query_stat_idx PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
+	query_hash		bigint CONSTRAINT aqo_query_stat_idx PRIMARY KEY REFERENCES public.aqo_queries ON DELETE CASCADE,
 	execution_time_with_aqo					double precision[],
 	execution_time_without_aqo				double precision[],
 	planning_time_with_aqo					double precision[],
@@ -27,7 +27,7 @@ CREATE TABLE public.aqo_query_stat (
 );
 
 CREATE TABLE public.aqo_data (
-	fspace_hash		int NOT NULL REFERENCES public.aqo_queries ON DELETE CASCADE,
+	fspace_hash		bigint NOT NULL REFERENCES public.aqo_queries ON DELETE CASCADE,
 	fsspace_hash	int NOT NULL,
 	nfeatures		int NOT NULL,
 	features		double precision[][],
@@ -52,12 +52,12 @@ CREATE TRIGGER aqo_queries_invalidate AFTER UPDATE OR DELETE OR TRUNCATE
 --
 
 -- Show query state at the AQO knowledge base
-CREATE FUNCTION public.aqo_status(hash int)
+CREATE FUNCTION public.aqo_status(hash bigint)
 RETURNS TABLE (
 	"learn"			BOOL,
 	"use aqo"		BOOL,
 	"auto tune"		BOOL,
-	"fspace hash"	INT,
+	"fspace hash"	bigINT,
 	"t_naqo"		TEXT,
 	"err_naqo"		TEXT,
 	"iters"			BIGINT,
@@ -87,7 +87,7 @@ WHERE (aqs.query_hash = aq.query_hash) AND
 	aqs.query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE FUNCTION public.aqo_enable_query(hash int)
+CREATE FUNCTION public.aqo_enable_query(hash bigint)
 RETURNS VOID
 AS $func$
 UPDATE public.aqo_queries SET
@@ -96,7 +96,7 @@ UPDATE public.aqo_queries SET
 	WHERE query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE FUNCTION public.aqo_disable_query(hash int)
+CREATE FUNCTION public.aqo_disable_query(hash bigint)
 RETURNS VOID
 AS $func$
 UPDATE public.aqo_queries SET
@@ -106,7 +106,7 @@ UPDATE public.aqo_queries SET
 	WHERE query_hash = $1;
 $func$ LANGUAGE SQL;
 
-CREATE FUNCTION public.aqo_clear_hist(hash int)
+CREATE FUNCTION public.aqo_clear_hist(hash bigint)
 RETURNS VOID
 AS $func$
 DELETE FROM public.aqo_data WHERE fspace_hash=$1;
@@ -120,7 +120,7 @@ SELECT query_hash FROM public.aqo_query_stat aqs
 	WHERE -1 = ANY (cardinality_error_with_aqo::double precision[]);
 $func$ LANGUAGE SQL;
 
-CREATE FUNCTION public.aqo_drop(hash int)
+CREATE FUNCTION public.aqo_drop(hash bigint)
 RETURNS VOID
 AS $func$
 DELETE FROM public.aqo_queries aq WHERE (aq.query_hash = $1);

aqo.c

@@ -8,18 +8,23 @@
  *	  aqo/aqo.c
  */
 
-#include "aqo.h"
-#include "cardinality_hooks.h"
-#include "ignorance.h"
-#include "path_utils.h"
-#include "new_storage.h"
+#include "postgres.h"
+#include "file_storage.h"
 
 #include "access/relation.h"
 #include "access/table.h"
 #include "catalog/pg_extension.h"
 #include "commands/extension.h"
+#include "miscadmin.h"
 #include "utils/selfuncs.h"
 
+#include "aqo.h"
+#include "cardinality_hooks.h"
+#include "ignorance.h"
+#include "path_utils.h"
+#include "preprocessing.h"
+
+
 PG_MODULE_MAGIC;
 
 void _PG_init(void);
@@ -28,6 +33,8 @@ void _PG_init(void);
 
 /* Strategy of determining feature space for new queries. */
 int		aqo_mode;
+bool	aqo_enabled = false; /* Signals that CREATE EXTENSION have executed and
+								all extension tables is ready for use. */
 bool	force_collect_stat;
 
 /*
@@ -124,14 +131,24 @@ aqo_free_callback(ResourceReleasePhase phase,
 
 	if (isTopLevel)
 	{
-		list_free(cur_classes);
+		list_free_deep(cur_classes);
 		cur_classes = NIL;
 	}
 }
 
 void
 _PG_init(void)
 {
+	/*
+	 * In order to create our shared memory area, we have to be loaded via
+	 * shared_preload_libraries.  If not, report an ERROR.
+	 */
+	if (!process_shared_preload_libraries_in_progress)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("AQO module could be loaded only on startup."),
+				 errdetail("Add 'aqo' into the shared_preload_libraries list.")));
+
 	DefineCustomEnumVariable("aqo.mode",
 							 "Mode of aqo usage.",
 							 NULL,
@@ -216,6 +233,8 @@ _PG_init(void)
 	ExecutorStart_hook							= aqo_ExecutorStart;
 	prev_ExecutorEnd_hook						= ExecutorEnd_hook;
 	ExecutorEnd_hook							= aqo_ExecutorEnd;
+
+	/* Cardinality prediction hooks. */
 	prev_set_baserel_rows_estimate_hook			= set_baserel_rows_estimate_hook;
 	set_foreign_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
 	set_baserel_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
@@ -225,17 +244,21 @@ _PG_init(void)
 	set_joinrel_size_estimates_hook				= aqo_set_joinrel_size_estimates;
 	prev_get_parameterized_joinrel_size_hook	= get_parameterized_joinrel_size_hook;
 	get_parameterized_joinrel_size_hook			= aqo_get_parameterized_joinrel_size;
+	prev_estimate_num_groups_hook				= estimate_num_groups_hook;
+	estimate_num_groups_hook					= aqo_estimate_num_groups_hook;
+	parampathinfo_postinit_hook					= ppi_hook;
+
 	prev_create_plan_hook						= create_plan_hook;
 	create_plan_hook							= aqo_create_plan_hook;
+
+	/* Service hooks. */
 	prev_ExplainOnePlan_hook					= ExplainOnePlan_hook;
 	ExplainOnePlan_hook							= print_into_explain;
 	prev_ExplainOneNode_hook					= ExplainOneNode_hook;
 	ExplainOneNode_hook							= print_node_explain;
-	parampathinfo_postinit_hook					= ppi_hook;
+
 	prev_create_upper_paths_hook				= create_upper_paths_hook;
 	create_upper_paths_hook						= aqo_store_upper_signature_hook;
-	prev_estimate_num_groups_hook				= estimate_num_groups_hook;
-	estimate_num_groups_hook					= aqo_estimate_num_groups_hook;
 
 	init_deactivated_queries_storage();
 	AQOMemoryContext = AllocSetContextCreate(TopMemoryContext,
@@ -319,3 +342,19 @@ init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2)
 	tag->locktag_type = LOCKTAG_USERLOCK;
 	tag->locktag_lockmethodid = USER_LOCKMETHOD;
 }
+
+/*
+ * AQO is really needed for any activity?
+ */
+bool
+IsQueryDisabled(void)
+{
+	if (!query_context.learn_aqo && !query_context.use_aqo &&
+		!query_context.auto_tuning && !query_context.collect_stat &&
+		!query_context.adding_query && !query_context.explain_only &&
+		INSTR_TIME_IS_ZERO(query_context.start_planning_time) &&
+		query_context.planning_time < 0.)
+		return true;
+
+	return false;
+}