Allow DB connection URLs to contain encoded reserved URL characters

Makefile

@@ -38,6 +38,7 @@ test-with-flags:
 
  @go test $(TEST_FLAGS) .
  @go test $(TEST_FLAGS) ./cli/...
+ @go test $(TEST_FLAGS) ./database
  @go test $(TEST_FLAGS) ./testing/...
 
  @echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{}

README.md

@@ -39,6 +39,25 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
  * [CockroachDB](database/cockroachdb)
  * [ClickHouse](database/clickhouse)
 
+### Database URLs
+
+Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false`
+
+Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character)
+
+Explicitly, the following characters need to be escaped:
+`!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]`
+
+It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python helpers below:
+```bash
+$ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$ python2 -c 'import urllib; print urllib.quote(raw_input("String to encode: "), "")'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$
+```
 
 ## Migration Sources
 

database/driver.go

@@ -81,7 +81,8 @@ type Driver interface {
 func Open(url string) (Driver, error) {
  u, err := nurl.Parse(url)
  if err != nil {
- return nil, err
+ return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+
+ "See: https:/golang-migrate/migrate#database-urls Error: %v", err)
  }
 
  if u.Scheme == "" {

database/mysql/mysql.go

@@ -13,8 +13,13 @@ import (
  nurl "net/url"
  "strconv"
  "strings"
+)
 
+import (
  "github.com/go-sql-driver/mysql"
+)
+
+import (
  "github.com/golang-migrate/migrate"
  "github.com/golang-migrate/migrate/database"
 )
@@ -89,8 +94,26 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
  return mx, nil
 }
 
+// urlToMySQLConfig takes a net/url URL and returns a go-sql-driver/mysql Config.
+// Manually sets username and password to avoid net/url from url-encoding the reserved URL characters
+func urlToMySQLConfig(u nurl.URL) (*mysql.Config, error) {
+ origUserInfo := u.User
+ u.User = nil
+
+ c, err := mysql.ParseDSN(strings.TrimPrefix(u.String(), "mysql://"))
+ if err != nil {
+ return nil, err
+ }
+ if origUserInfo != nil {
+ c.User = origUserInfo.Username()
+ if p, ok := origUserInfo.Password(); ok {
+ c.Passwd = p
+ }
+ }
+ return c, nil
+}
+
 func (m *Mysql) Open(url string) (database.Driver, error) {
- url = strings.TrimPrefix(url, "mysql://")
  purl, err := nurl.Parse(url)
  if err != nil {
  return nil, err
@@ -100,7 +123,11 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
  q.Set("multiStatements", "true")
  purl.RawQuery = q.Encode()
 
- db, err := sql.Open("mysql", migrate.FilterCustomQuery(purl).String())
+ c, err := urlToMySQLConfig(*migrate.FilterCustomQuery(purl))
+ if err != nil {
+ return nil, err
+ }
+ db, err := sql.Open("mysql", c.FormatDSN())
  if err != nil {
  return nil, err
  }

database/mysql/mysql_test.go

@@ -4,11 +4,15 @@ import (
  "database/sql"
  sqldriver "database/sql/driver"
  "fmt"
- // "io/ioutil"
- // "log"
+ "net/url"
  "testing"
+)
 
+import (
  "github.com/go-sql-driver/mysql"
+)
+
+import (
  dt "github.com/golang-migrate/migrate/database/testing"
  mt "github.com/golang-migrate/migrate/testing"
 )
@@ -97,3 +101,55 @@ func TestLockWorks(t *testing.T) {
  }
  })
 }
+
+func TestURLToMySQLConfig(t *testing.T) {
+ testcases := []struct {
+ name string
+ urlStr string
+ expectedDSN string // empty string signifies that an error is expected
+ }{
+ {name: "no user/password", urlStr: "mysql://tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "only user", urlStr: "mysql://username@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "only user - with encoded :",
+ urlStr: "mysql://username%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "only user - with encoded @",
+ urlStr: "mysql://username%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "user/password", urlStr: "mysql://username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ // Not supported yet: https:/go-sql-driver/mysql/issues/591
+ // {name: "user/password - user with encoded :",
+ // urlStr: "mysql://username%3A:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ // expectedDSN: "username::pasword@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "user/password - user with encoded @",
+ urlStr: "mysql://username%40:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username@:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "user/password - password with encoded :",
+ urlStr: "mysql://username:password%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username:password:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ {name: "user/password - password with encoded @",
+ urlStr: "mysql://username:password%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+ expectedDSN: "username:password@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+ }
+ for _, tc := range testcases {
+ t.Run(tc.name, func(t *testing.T) {
+ u, err := url.Parse(tc.urlStr)
+ if err != nil {
+ t.Fatal("Failed to parse url string:", tc.urlStr, "error:", err)
+ }
+ if config, err := urlToMySQLConfig(*u); err == nil {
+ dsn := config.FormatDSN()
+ if dsn != tc.expectedDSN {
+ t.Error("Got unexpected DSN:", dsn, "!=", tc.expectedDSN)
+ }
+ } else {
+ if tc.expectedDSN != "" {
+ t.Error("Got unexpected error:", err, "urlStr:", tc.urlStr)
+ }
+ }
+ })
+ }
+}