Skip to content

Commit

Permalink
Merge pull request #540 from Nitnelav/4.X
Browse files Browse the repository at this point in the history 
update & cleanup matsim experimental
The most notable change is the way time data is stored.
it was done as a string (like '1h_2h', '2h_3h', etc.). Now it is fully configurable as an integer representing the timeBin
  • Loading branch information
@nicolas-f
nicolas-f authored Feb 6, 2023
2 parents 35e207a + 2b3a1aa commit 68805dd
Show file tree
Hide file tree
Showing 7 changed files with 992 additions and 924 deletions.
2 changes: 1 addition & 1 deletion wps_scripts/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ dependencies {
exclude group: 'org.geotools'
}

implementation 'com.opencsv:opencsv:4.0'
implementation 'com.opencsv:opencsv:5.7.1'
implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.32'
testCompileOnly 'junit:junit:4.12'
}
Expand Down
594 changes: 294 additions & 300 deletions ...main/groovy/org/noise_planet/noisemodelling/wps/Experimental_Matsim/Agent_Exposure.groovy
View file

Large diffs are not rendered by default.

240 changes: 161 additions & 79 deletions .../noise_planet/noisemodelling/wps/Experimental_Matsim/Noise_From_Attenuation_Matrix.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ package org.noise_planet.noisemodelling.wps.Experimental_Matsim

import geoserver.GeoServer
import geoserver.catalog.Store
import groovy.sql.GroovyRowResult
import org.geotools.jdbc.JDBCDataStore
import org.h2gis.utilities.wrapper.ConnectionWrapper
import org.locationtech.jts.geom.Geometry
import org.slf4j.Logger
import org.slf4j.LoggerFactory

Expand All @@ -37,22 +39,21 @@ inputs = [
'<br/>The table must contain the following fields : (PK, LINK_ID, THE_GEOM)',
type: String.class
],
matsimRoadsStats : [
name: 'Table name of the MATSIM table containing the roads LW stats per timeString',
title: 'Table name of the MATSIM table containing the roads LW stats per timeString',
description: 'Table name of the MATSIM table containing the roads LW stats per timeString' +
matsimRoadsLw : [
name: 'Table name of the MATSIM table containing the roads LW stats per timeBin',
title: 'Table name of the MATSIM table containing the roads LW stats per timeBin',
description: 'Table name of the MATSIM table containing the roads LW stats per timeBin' +
'<br/>The table must contain the following fields : ' +
'<br/>PK, LINK_ID, LW63, LW125, LW250, LW500, LW1000, LW2000, LW4000, LW8000, TIMESTRING',
'<br/>PK, LINK_ID, LW63, LW125, LW250, LW500, LW1000, LW2000, LW4000, LW8000, TIME',
type: String.class
],
timeString: [
name: 'TIMESTRING Field value',
title: 'TIMESTRING Field value.',
description: 'TIMESTRING Field value. If defined will only output data for the specified timeString.' +
'<br/>The timeString can be "D", "E", "N" for DEN analysis, "12_14" for example for hour analysis or "0h15_0h30" for example for 15minutes analysis.',
min: 0,
max : 1,
type: String.class
receiversTable : [
name: 'Name of the table containing the receivers',
title: 'Name of the table containing the receivers',
description: 'Name of the table containing the receivers' +
'<br/>The table must contain the following fields : ' +
'<br/>PK, THE_GEOM',
type: String.class
],
attenuationTable : [
name: 'Attenuation Matrix Table name',
Expand All @@ -62,12 +63,21 @@ inputs = [
'<br/>IDRECEIVER, IDSOURCE, THE_GEOM, HZ63, HZ125, HZ250, HZ500, HZ1000, HZ2000, HZ4000, HZ8000',
type: String.class
],
timeBinSize: [
name: 'The size of time bins in seconds.',
title: 'The size of time bins in seconds.',
description: 'This parameter dictates the time resolution of the resulting data ' +
'<br/>The time information stored will be the starting time of the time bins ' +
'<br/>For exemple with a timeBinSize of 3600, the data will be analysed using the following timeBins: ' +
'<br/>0, 3600, 7200, ..., 79200, 82800',
type: Integer.class
],
outTableName: [
name: 'Output table name',
title: 'Output table name',
description: 'Output table name' +
'<br/>The table will contain the following fields :' +
'<br/>PK, IDRECEIVER, THE_GEOM, HZ63, HZ125, HZ250, HZ500, HZ1000, HZ2000, HZ000, HZ8000, TIMESTRING',
'<br/>PK, IDRECEIVER, THE_GEOM, HZ63, HZ125, HZ250, HZ500, HZ1000, HZ2000, HZ000, HZ8000, TIME',
type: String.class
]
]
Expand Down Expand Up @@ -117,83 +127,115 @@ def exec(Connection connection, input) {
logger.info("inputs {}", input)

String matsimRoads = input['matsimRoads']
String matsimRoadsStats = input['matsimRoadsStats']

String timeString = ""
if (input["timeString"]) {
timeString = input["timeString"];
}

String matsimRoadsLw = input['matsimRoadsLw']
String attenuationTable = input['attenuationTable']
String receiversTable = input['receiversTable']
String outTableName = input['outTableName']

int timeBinSize = 3600
if (input["timeBinSize"]) {
timeBinSize = input["timeBinSize"] as int;
}

DatabaseMetaData dbMeta = connection.getMetaData();
ResultSet rs = dbMeta.getIndexInfo(null, null, attenuationTable, false, false);


sql.execute(String.format("DROP TABLE %s IF EXISTS", outTableName))
String query = "CREATE TABLE " + outTableName + '''(
PK integer PRIMARY KEY AUTO_INCREMENT,
IDRECEIVER integer,
THE_GEOM geometry,
HZ63 double precision,
HZ125 double precision,
HZ250 double precision,
HZ500 double precision,
HZ1000 double precision,
HZ2000 double precision,
HZ4000 double precision,
HZ8000 double precision,
TIME int
)
'''
sql.execute(query)
PreparedStatement insert_stmt = connection.prepareStatement(
"INSERT INTO " + outTableName + " VALUES(DEFAULT, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
)

logger.info("searching indexes on attenuation matrix ... ")
boolean indexIDSOURCE = false;
boolean indexGEOM = false;
while (rs.next()) {
String column = rs.getString("COLUMN_NAME");
String pos = rs.getString("ORDINAL_POSITION");
if (column == "IDSOURCE" && pos == "1") {
indexIDSOURCE = true;
logger.info("index on attenuation matrix IDSOURCE found")
ensureIndex(connection, attenuationTable, "IDSOURCE", false)
ensureIndex(connection, attenuationTable, "IDRECEIVER", false)
logger.info("searching indexes on traffic tables ... ")
ensureIndex(connection, matsimRoads, "LINK_ID", false)
ensureIndex(connection, matsimRoadsLw, "LINK_ID", false)
ensureIndex(connection, matsimRoadsLw, "TIME", false)

List<String> mrs_freqs = ["LW63", "LW125", "LW250", "LW500", "LW1000", "LW2000", "LW4000", "LW8000"]

long count = 0, do_print = 1
List<GroovyRowResult> receivers_res = sql.rows("SELECT * FROM " + receiversTable);
long nb_receivers = receivers_res.size()
long start = System.currentTimeMillis();
for (GroovyRowResult receiver: receivers_res) {
long receiver_id = receiver["PK"] as long;
Geometry receiver_geom = receiver["THE_GEOM"] as Geometry;
Map<Integer, List<Double>> levels = new HashMap<Integer, List<Double>>();
List<GroovyRowResult> sources_att_res = sql.rows(String.format("SELECT lg.* FROM %s lg WHERE lg.IDRECEIVER = %d", attenuationTable, receiver_id));
long nb_sources = sources_att_res.size();
if (nb_sources == 0) {
count++
continue
}
if (column == "THE_GEOM" && pos == "1") {
indexGEOM = true;
logger.info("index on attenuation matrix THE_GEOM found")
for (GroovyRowResult sources_att: sources_att_res) {
long source_id = sources_att["IDSOURCE"] as long;
List<Double> attenuation = [
sources_att["HZ63"] as double,
sources_att["HZ125"] as double,
sources_att["HZ250"] as double,
sources_att["HZ500"] as double,
sources_att["HZ1000"] as double,
sources_att["HZ2000"] as double,
sources_att["HZ4000"] as double,
sources_att["HZ8000"] as double,
];
List<GroovyRowResult> roads_stats_res = sql.rows(String.format(
"SELECT mrs.* FROM %s mrs INNER JOIN %s mr ON mr.LINK_ID = mrs.LINK_ID WHERE mr.PK = %d",
matsimRoadsLw, matsimRoads, source_id));
for (GroovyRowResult roads_stats: roads_stats_res) {
int timeBin = roads_stats["TIME"] as int
if (!levels.containsKey(timeBin)) {
levels[timeBin] = [-99.0, -99.0, -99.0, -99.0, -99.0, -99.0, -99.0, -99.0] as List<Double>
}
for (i in 0..<8) {
double new_level = (roads_stats[mrs_freqs[i]] as double) + attenuation[i];
levels[timeBin][i] = 10 * Math.log10( Math.pow(10, levels[timeBin][i] / 10) + Math.pow(10, new_level / 10))
}
}
}
}

if (!indexIDSOURCE) {
logger.info("index on attenuation matrix IDSOURCE, NOT found, creating one...")
sql.execute("CREATE INDEX ON " + attenuationTable + " (IDSOURCE)");
}
if (!indexGEOM) {
logger.info("index on attenuation matrix THE_GEOM, NOT found, creating one...")
sql.execute("CREATE SPATIAL INDEX ON " + attenuationTable + " (THE_GEOM)");
for (int timeBin = 0; timeBin < 86400; timeBin += timeBinSize) {
if (!levels.containsKey(timeBin)) {
levels[timeBin] = [-99.0, -99.0, -99.0, -99.0, -99.0, -99.0, -99.0, -99.0] as List<Double>
}
List<Double> ts_levels = levels[timeBin]
insert_stmt.setLong(1, receiver_id)
insert_stmt.setString(2, receiver_geom.toText())
for (i in 0..<8) {
insert_stmt.setDouble(i+3, ts_levels[i])
}
insert_stmt.setInt(11, timeBin)
insert_stmt.execute()
}
if (count >= do_print) {
double elapsed = (System.currentTimeMillis() - start + 1) / 1000
logger.info(String.format("Processing Receiver %d (max:%d) - elapsed : %ss (%.1fit/s)",
count, nb_receivers, elapsed, count/elapsed))
do_print *= 2
}
count ++
}

sql.execute(String.format("DROP TABLE IF EXISTS %s", outTableName))

String query = "CREATE TABLE " + outTableName + '''(
PK integer PRIMARY KEY AUTO_INCREMENT,
IDRECEIVER integer,
THE_GEOM geometry,
HZ63 double precision,
HZ125 double precision,
HZ250 double precision,
HZ500 double precision,
HZ1000 double precision,
HZ2000 double precision,
HZ4000 double precision,
HZ8000 double precision,
TIMESTRING varchar
);
INSERT INTO RESULT_GEOM(IDRECEIVER , THE_GEOM , HZ63 , HZ125 , HZ250 , HZ500 , HZ1000 , HZ2000 , HZ4000 , HZ8000 , TIMESTRING )
SELECT lg.IDRECEIVER, lg.THE_GEOM,
10 * LOG10( SUM(POWER(10,(mrs.LW63 + lg.HZ63) / 10))) AS HZ63,
10 * LOG10( SUM(POWER(10,(mrs.LW125 + lg.HZ125) / 10))) AS HZ125,
10 * LOG10( SUM(POWER(10,(mrs.LW250 + lg.HZ250) / 10))) AS HZ250,
10 * LOG10( SUM(POWER(10,(mrs.LW500 + lg.HZ500) / 10))) AS HZ500,
10 * LOG10( SUM(POWER(10,(mrs.LW1000 + lg.HZ1000) / 10))) AS HZ1000,
10 * LOG10( SUM(POWER(10,(mrs.LW2000 + lg.HZ2000) / 10))) AS HZ2000,
10 * LOG10( SUM(POWER(10,(mrs.LW4000 + lg.HZ4000) / 10))) AS HZ4000,
10 * LOG10( SUM(POWER(10,(mrs.LW8000 + lg.HZ8000) / 10))) AS HZ8000,
mrs.TIMESTRING AS TIMESTRING
FROM ''' + attenuationTable + ''' lg
INNER JOIN ''' + matsimRoads + ''' mr ON lg.IDSOURCE = mr.PK
INNER JOIN ''' + matsimRoadsStats + ''' mrs ON mr.LINK_ID = mrs.LINK_ID
''' + ((timeString != "") ? "WHERE mrs.TIMESTRING = \'" + timeString + "\' " : "") + '''
GROUP BY lg.IDRECEIVER, lg.THE_GEOM, mrs.TIMESTRING;
'''

logger.info(query)

sql.execute(query)

prefix = "HZ"
String prefix = "HZ"
sql.execute("ALTER TABLE " + outTableName + " ADD COLUMN LEQA float as 10*log10((power(10,(" + prefix + "63-26.2)/10)+power(10,(" + prefix + "125-16.1)/10)+power(10,(" + prefix + "250-8.6)/10)+power(10,(" + prefix + "500-3.2)/10)+power(10,(" + prefix + "1000)/10)+power(10,(" + prefix + "2000+1.2)/10)+power(10,(" + prefix + "4000+1)/10)+power(10,(" + prefix + "8000-1.1)/10)))")
sql.execute("ALTER TABLE " + outTableName + " ADD COLUMN LEQ float as 10*log10((power(10,(" + prefix + "63)/10)+power(10,(" + prefix + "125)/10)+power(10,(" + prefix + "250)/10)+power(10,(" + prefix + "500)/10)+power(10,(" + prefix + "1000)/10)+power(10,(" + prefix + "2000)/10)+power(10,(" + prefix + "4000)/10)+power(10,(" + prefix + "8000)/10)))")

Expand All @@ -203,3 +245,43 @@ def exec(Connection connection, input) {
return resultString
}

static boolean tableExists(Connection connection, String table) {
DatabaseMetaData dbMeta = connection.getMetaData();
ResultSet rs = dbMeta.getTables(null, null, table, null);
boolean table_found = false;
if (rs.next()) {
table_found = true
}
return table_found
}

static boolean columnExists(Connection connection, String table, String column_name) {
DatabaseMetaData dbMeta = connection.getMetaData();
ResultSet rs = dbMeta.getColumns(null, null, table, column_name);
boolean col_found = false;
if (rs.next()) {
col_found = true
}
return col_found
}

static boolean indexExists(Connection connection, String table, String column_name) {
DatabaseMetaData dbMeta = connection.getMetaData();
ResultSet rs = dbMeta.getIndexInfo(null, null, table, false, false);
boolean index_found = false;
while (rs.next()) {
String column = rs.getString("COLUMN_NAME");
String pos = rs.getString("ORDINAL_POSITION");
if (column == column_name && pos == "1") {
index_found = true;
}
}
return index_found
}

static void ensureIndex(Connection connection, String table, String column_name, boolean spatial) {
if (!indexExists(connection, table, column_name)) {
Sql sql = new Sql(connection)
sql.execute("CREATE " + (spatial ? "SPATIAL " : "") + "INDEX ON " + table + " (" + column_name + ")");
}
}
Loading

0 comments on commit 68805dd

Please sign in to comment.