Stable13

Makefile

@@ -2,25 +2,31 @@
 
 EXTENSION = aqo
 EXTVERSION = 1.2
-PGFILEDESC = "AQO - adaptive query optimization"
-MODULES = aqo
+PGFILEDESC = "AQO - Adaptive Query Optimization"
+MODULE_big = aqo
 OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
 hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
-selectivity_cache.o storage.o utils.o $(WIN32RES)
+selectivity_cache.o storage.o utils.o ignorance.o $(WIN32RES)
+
+TAP_TESTS = 1
 
 REGRESS =	aqo_disabled \
 			aqo_controlled \
 			aqo_intelligent \
 			aqo_forced \
 			aqo_learn \
 			schema \
-			aqo_CVE-2020-14350
+			aqo_fdw \
+			aqo_CVE-2020-14350 \
+			gucs
 
+fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
+PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir)
 EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add
+EXTRA_INSTALL = contrib/postgres_fdw
 
 DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql
 
-MODULE_big = aqo
 ifdef USE_PGXS
 PG_CONFIG ?= pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)

README.md

@@ -9,6 +9,11 @@ complicated queries.
 ## Installation
 
 The module works with PostgreSQL 9.6 and above.
+To avoid compatibility issues, the following branches in the git-repository are allocated:
+* `stable9_6`.
+* `stable11` - for PG v10 and v11.
+* `stable12` - for PG v12.
+* the `master` branch of the AQO repository correctly works with PGv13 and the PostgreSQL `master` branch.
 
 The module contains a patch and an extension. Patch has to be applied to the
 sources of PostgresSQL. Patch affects header files, that is why PostgreSQL
@@ -19,7 +24,7 @@ installed with `make install`.
 
 ```
 cd postgresql-9.6                                                # enter postgresql source directory
-git clone https://github.com/tigvarts/aqo.git contrib/aqo        # clone aqo into contrib
+git clone https://github.com/postgrespro/aqo.git contrib/aqo        # clone aqo into contrib
 patch -p1 --no-backup-if-mismatch < contrib/aqo/aqo_pg<version>.patch  # patch postgresql
 make clean && make && make install                               # recompile postgresql
 cd contrib/aqo                                                   # enter aqo directory
@@ -28,7 +33,7 @@ make check                                              # check whether it works
 ```
 
 Tag `version` at the patch name corresponds to suitable PostgreSQL release.
-For PostgreSQL 10 use aqo_pg10.patch; for PostgreSQL 11 use aqo_pg11.patch and so on.
+For PostgreSQL 9.6 use the 'aqo_pg9_6.patch' file; PostgreSQL 10 use aqo_pg10.patch; for PostgreSQL 11 use aqo_pg11.patch and so on.
 Also, you can see git tags at the master branch for more accurate definition of
 suitable PostgreSQL version.
 
@@ -50,7 +55,7 @@ of per-database.
 
 The typical case is follows: you have complicated query, which executes too
 long. `EXPLAIN ANALYZE` shows, that the possible reason is bad cardinality
-estimnation.
+estimation.
 
 Example:
 ```
@@ -127,16 +132,16 @@ When the plan stops changing, you can often observe performance improvement:
 (23 rows)
 ```
 
-The settings system in AQO works with normalized queries, i. e. queries with
-removed constants. For example, the normalized version of
+The settings system in AQO works with normalised queries, i. e. queries with
+removed constants. For example, the normalised version of
 `SELECT * FROM tbl WHERE a < 25 AND b = 'str';`
 is
 `SELECT * FROM tbl WHERE a < CONST and b = CONST;`
 
-So the queries have equal normalization if and only if they differ only
+So the queries have equal normalisation if and only if they differ only
 in their constants.
 
-Each normalized query has its own hash. The correspondence between normalized
+Each normalised query has its own hash. The correspondence between normalised
 query hash and query text is stored in aqo_query_texts table:
 ```
 SELECT * FROM aqo_query_texts;
@@ -174,6 +179,10 @@ if the data tends to change significantly), you can do
 `UPDATE SET aqo_learn=false WHERE query_hash = <query_hash>;`
 before commit.
 
+The extension includes two GUC's to display the executed cardinality predictions for a query.
+The `aqo.show_details = 'on'` (default - off) allows to see the aqo cardinality prediction results for each node of a query plan and an AQO summary.
+The `aqo.show_hash = 'on'` (default - off) will print hash signature for each plan node and overall query. It is system-specific information and should be used for situational analysis.
+
 The more detailed reference of AQO settings mechanism is available further.
 
 ## Advanced tuning

aqo.c

@@ -2,23 +2,43 @@
  * aqo.c
  *		Adaptive query optimization extension
  *
- * Copyright (c) 2016-2020, Postgres Professional
+ * Copyright (c) 2016-2021, Postgres Professional
  *
  * IDENTIFICATION
  *	  aqo/aqo.c
  */
 
 #include "aqo.h"
+#include "ignorance.h"
+
+#include "access/relation.h"
+#include "access/table.h"
+#include "catalog/pg_extension.h"
+#include "commands/extension.h"
 
 PG_MODULE_MAGIC;
 
 void _PG_init(void);
 
+#define AQO_MODULE_MAGIC	(1234)
 
 /* Strategy of determining feature space for new queries. */
 int		aqo_mode;
 bool	force_collect_stat;
 
+/*
+ * Show special info in EXPLAIN mode.
+ *
+ * aqo_show_hash - show query class (hash) and a feature space value (hash)
+ * of each plan node. This is instance-dependent value and can't be used
+ * in regression and TAP tests.
+ *
+ * aqo_show_details - show AQO settings for this class and prediction
+ * for each plan node.
+ */
+bool	aqo_show_hash;
+bool	aqo_show_details;
+
 /* GUC variables */
 static const struct config_enum_entry format_options[] = {
 	{"intelligent", AQO_MODE_INTELLIGENT, false},
@@ -76,12 +96,14 @@ post_parse_analyze_hook_type				prev_post_parse_analyze_hook;
 planner_hook_type							prev_planner_hook;
 ExecutorStart_hook_type						prev_ExecutorStart_hook;
 ExecutorEnd_hook_type						prev_ExecutorEnd_hook;
+set_baserel_rows_estimate_hook_type			prev_set_foreign_rows_estimate_hook;
 set_baserel_rows_estimate_hook_type			prev_set_baserel_rows_estimate_hook;
 get_parameterized_baserel_size_hook_type	prev_get_parameterized_baserel_size_hook;
 set_joinrel_size_estimates_hook_type		prev_set_joinrel_size_estimates_hook;
 get_parameterized_joinrel_size_hook_type	prev_get_parameterized_joinrel_size_hook;
 copy_generic_path_info_hook_type			prev_copy_generic_path_info_hook;
 ExplainOnePlan_hook_type					prev_ExplainOnePlan_hook;
+ExplainOneNode_hook_type					prev_ExplainOneNode_hook;
 
 /*****************************************************************************
  *
@@ -102,7 +124,8 @@ _PG_init(void)
 							 0,
 							 NULL,
 							 NULL,
-							 NULL);
+							 NULL
+	);
 
 	DefineCustomBoolVariable(
 							 "aqo.force_collect_stat",
@@ -115,7 +138,46 @@ _PG_init(void)
 							 NULL,
 							 NULL,
 							 NULL
-		);
+	);
+
+	DefineCustomBoolVariable(
+							 "aqo.show_hash",
+							 "Show query and node hash on explain.",
+							 "Hash value depend on each instance and is not good to enable it in regression or TAP tests.",
+							 &aqo_show_hash,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL
+	);
+
+	DefineCustomBoolVariable(
+							 "aqo.show_details",
+							 "Show AQO state on a query.",
+							 NULL,
+							 &aqo_show_details,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL
+	);
+
+	DefineCustomBoolVariable(
+							 "aqo.log_ignorance",
+							 "Log in a special table all feature spaces for which the AQO prediction was not successful.",
+							 NULL,
+							 &aqo_log_ignorance,
+							 false,
+							 PGC_SUSET,
+							 0,
+							 NULL,
+							 set_ignorance,
+							 NULL
+	);
 
 	prev_planner_hook							= planner_hook;
 	planner_hook								= aqo_planner;
@@ -126,6 +188,7 @@ _PG_init(void)
 	prev_ExecutorEnd_hook						= ExecutorEnd_hook;
 	ExecutorEnd_hook							= aqo_ExecutorEnd;
 	prev_set_baserel_rows_estimate_hook			= set_baserel_rows_estimate_hook;
+	set_foreign_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
 	set_baserel_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
 	prev_get_parameterized_baserel_size_hook	= get_parameterized_baserel_size_hook;
 	get_parameterized_baserel_size_hook			= aqo_get_parameterized_baserel_size;
@@ -137,6 +200,8 @@ _PG_init(void)
 	copy_generic_path_info_hook					= aqo_copy_generic_path_info;
 	prev_ExplainOnePlan_hook					= ExplainOnePlan_hook;
 	ExplainOnePlan_hook							= print_into_explain;
+	prev_ExplainOneNode_hook					= ExplainOneNode_hook;
+	ExplainOneNode_hook							= print_node_explain;
 	parampathinfo_postinit_hook					= ppi_hook;
 
 	init_deactivated_queries_storage();
@@ -158,3 +223,63 @@ invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS)
 	init_deactivated_queries_storage();
 	PG_RETURN_POINTER(NULL);
 }
+
+/*
+ * Return AQO schema's Oid or InvalidOid if that's not possible.
+ */
+Oid
+get_aqo_schema(void)
+{
+	Oid				result;
+	Relation		rel;
+	SysScanDesc		scandesc;
+	HeapTuple		tuple;
+	ScanKeyData		entry[1];
+	Oid				ext_oid;
+
+	/* It's impossible to fetch pg_aqo's schema now */
+	if (!IsTransactionState())
+		return InvalidOid;
+
+	ext_oid = get_extension_oid("aqo", true);
+	if (ext_oid == InvalidOid)
+		return InvalidOid; /* exit if pg_aqo does not exist */
+
+	ScanKeyInit(&entry[0],
+#if PG_VERSION_NUM >= 120000
+				Anum_pg_extension_oid,
+#else
+				ObjectIdAttributeNumber,
+#endif
+				BTEqualStrategyNumber, F_OIDEQ,
+				ObjectIdGetDatum(ext_oid));
+
+	rel = relation_open(ExtensionRelationId, AccessShareLock);
+	scandesc = systable_beginscan(rel, ExtensionOidIndexId, true,
+								  NULL, 1, entry);
+	tuple = systable_getnext(scandesc);
+
+	/* We assume that there can be at most one matching tuple */
+	if (HeapTupleIsValid(tuple))
+		result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace;
+	else
+		result = InvalidOid;
+
+	systable_endscan(scandesc);
+	relation_close(rel, AccessShareLock);
+	return result;
+}
+
+/*
+ * Init userlock
+ */
+void
+init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2)
+{
+	tag->locktag_field1 = AQO_MODULE_MAGIC;
+	tag->locktag_field2 = key1;
+	tag->locktag_field3 = key2;
+	tag->locktag_field4 = 0;
+	tag->locktag_type = LOCKTAG_USERLOCK;
+	tag->locktag_lockmethodid = USER_LOCKMETHOD;
+}