Fix orphaned objects deletion bug (#15657) (#15682)

* Fix orphaned objects deletion bug

* extend test

Co-authored-by: 6543 <6543@obermui.de>

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

.drone.yml

@@ -113,18 +113,6 @@ services:
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: yes
       MYSQL_DATABASE: test
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-      GITLAB_READ_TOKEN:
-        from_secret: gitlab_read_token
-    depends_on:
-      - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
 
   - name: mysql8
     pull: default
@@ -678,7 +666,6 @@ steps:
       event:
         exclude:
         - pull_request
-
 ---
 kind: pipeline
 name: docker-linux-arm64-dry-run
@@ -708,6 +695,9 @@ steps:
       tags: linux-arm64
       build_args:
         - GOPROXY=off
+    environment:
+      PLUGIN_MIRROR:
+        from_secret: plugin_mirror
     when:
       event:
         - pull_request
@@ -752,11 +742,13 @@ steps:
         from_secret: docker_password
       username:
         from_secret: docker_username
+    environment:
+      PLUGIN_MIRROR:
+        from_secret: plugin_mirror
     when:
       event:
         exclude:
         - pull_request
-
 ---
 kind: pipeline
 name: docker-manifest

CHANGELOG.md

@@ -4,14 +4,167 @@ This changelog goes through all the changes that have been made in each release
 without substantial changes to our git log; to see the highlights of what has
 been added to each release, please refer to the [blog](https://blog.gitea.io).
 
-## [1.13.0-RC1](https://github.com/go-gitea/gitea/releases/tag/v1.13.0-RC1) - 2020-10-14
+## [1.13.7](https://github.com/go-gitea/gitea/releases/tag/v1.13.7) - 2021-04-07
 
 * SECURITY
+  * Update to bluemonday-1.0.6 (#15294) (#15298)
+  * Clusterfuzz found another way (#15160) (#15169)
+* API
+  * Fix wrong user returned in API (#15139) (#15150)
+* BUGFIXES
+  * Add 'fonts' into 'KnownPublicEntries' (#15188) (#15317)
+  * Speed up `enry.IsVendor` (#15213) (#15246)
+  * Response 404 for diff/patch of a commit that not exist (#15221) (#15238)
+  * Prevent NPE in CommentMustAsDiff if no hunk header (#15199) (#15201)
+* MISC
+  * Add size to Save function (#15264) (#15271)
+
+## [1.13.6](https://github.com/go-gitea/gitea/releases/tag/v1.13.6) - 2021-03-23
+
+* SECURITY
+  * Fix bug on avatar middleware (#15124) (#15125)
+  * Fix another clusterfuzz identified issue (#15096) (#15114)
+* API
+  * Fix nil exeption for get pull reviews API #15104 (#15106)
+* BUGFIXES
+  * Fix markdown rendering in milestone content (#15056) (#15092)
+
+## [1.13.5](https://github.com/go-gitea/gitea/releases/tag/v1.13.5) - 2021-03-21
+
+* SECURITY
+  * Update to goldmark 1.3.3 (#15059) (#15061)
+* API
+  * Fix set milestone on PR creation (#14981) (#15001)
+  * Prevent panic when editing forked repos by API (#14960) (#14963)
+* BUGFIXES
+  * Fix bug when upload on web (#15042) (#15055)
+  * Delete Labels & IssueLabels on Repo Delete too (#15039) (#15051)
+  * another clusterfuzz spotted issue (#15032) (#15034)
+  * Fix postgres ID sequences broken by recreate-table (#15015) (#15029)
+  * Fix several render issues (#14986) (#15013)
+  * Make sure sibling images get a link too (#14979) (#14995)
+  * Fix Anchor jumping with escaped query components (#14969) (#14977)
+  * fix release mail html template (#14976)
+  * Fix excluding more than two labels on issues list (#14962) (#14973)
+  * don't mark each comment poster as OP (#14971) (#14972)
+  * Add "captcha" to list of reserved usernames (#14930)
+  * Re-enable import local paths after reversion from #13610 (#14925) (#14927)
+
+## [1.13.4](https://github.com/go-gitea/gitea/releases/tag/v1.13.4) - 2021-03-07
+
+* SECURITY
+  * Fix issue popups (#14898) (#14899)
+* BUGFIXES
+  * Fix race in LFS ContentStore.Put(...) (#14895) (#14913)
+  * Fix a couple of issues with a feeds (#14897) (#14903)
+  * When transfering repository and database transaction failed, rollback the renames (#14864) (#14902)
+  * Fix race in local storage (#14888) (#14901)
+  * Fix 500 on pull view page if user is not loged in (#14885) (#14886)
+* DOCS
+  * Fix how lfs data path is set (#14855) (#14884)
+
+## [1.13.3](https://github.com/go-gitea/gitea/releases/tag/v1.13.3) - 2021-03-04
+
+* BREAKING
+  * Turn default hash password algorithm back to pbkdf2 from argon2 until we find a better one (#14673) (#14675)
+* BUGFIXES
+  * Fix paging of file commit logs (#14831) (#14879)
+  * Print useful error if SQLite is used in settings but not supported (#14476) (#14874)
+  * Fix display since time round (#14226) (#14873)
+  * When Deleting Repository only explicitly close PRs whose base is not this repository (#14823) (#14842)
+  * Set HCaptchaSiteKey on Link Account pages (#14834) (#14839)
+  * Fix a couple of CommentAsPatch issues.  (#14804) (#14820)
+  * Disable broken OAuth2 providers at startup (#14802) (#14811)
+  * Repo Transfer permission checks (#14792) (#14794)
+  * Fix double alert in oauth2 application edit view (#14764) (#14768)
+  * Fix broken spans in diffs (#14678) (#14683)
+  * Prevent race in PersistableChannelUniqueQueue.Has (#14651) (#14676)
+  * HasPreviousCommit causes recursive load of commits unnecessarily (#14598) (#14649)
+  * Do not assume all 40 char strings are SHA1s (#14624) (#14648)
+  * Allow org labels to be set with issue templates (#14593) (#14647)
+  * Accept multiple SSH keys in single LDAP SSHPublicKey attribute (#13989) (#14607)
+  * Fix bug about ListOptions and stars/watchers pagnation (#14556) (#14573)
+  * Fix GPG key deletion during account deletion (#14561) (#14569)
+
+## [1.13.2](https://github.com/go-gitea/gitea/releases/tag/v1.13.2) - 2021-01-31
+
+* SECURITY
+  * Prevent panic on fuzzer provided string (#14405) (#14409)
+  * Add secure/httpOnly attributes to the lang cookie (#14279) (#14280)
+* API
+  * If release publisher is deleted use ghost user (#14375)
+* BUGFIXES
+  * Internal ssh server respect Ciphers, MACs and KeyExchanges settings (#14523) (#14530)
+  * Set the name Mapper in migrations (#14526) (#14529)
+  * Fix wiki preview (#14515)
+  * Update code.gitea.io/sdk/gitea v0.13.1 -> v0.13.2 (#14497)
+  * ChangeUserName: rename user files back on DB issue (#14447)
+  * Fix lfs preview bug (#14428) (#14433)
+  * Ensure timeout error is shown on u2f timeout (#14417) (#14431)
+  * Fix Deadlock & Delete affected reactions on comment deletion (#14392) (#14425)
+  * Use path not filepath in routers/editor (#14390) (#14396)
+  * Check if label template exist first (#14384) (#14389)
+  * Fix migration v141 (#14387) (#14388)
+  * Use Request.URL.RequestURI() for fcgi (#14347)
+  * Use ServerError provided by Context (#14333) (#14345)
+  * Fix edit-label form init (#14337)
+  * Fix mailIssueCommentBatch for pull request (#14252) (#14296)
+  * Render links for commit hashes followed by comma (#14224) (#14227)
+  * Send notifications for mentions in pulls, issues, (code-)comments (#14218) (#14221)
+  * Fix avatar bugs (#14217) (#14220)
+  * Ensure that schema search path is set with every connection on postgres (#14131) (#14216)
+  * Fix dashboard issues labels filter bug (#14210) (#14214)
+  * When visit /favicon.ico but the static file is not exist return 404 but not continue to handle the route (#14211) (#14213)
+  * Fix branch selector on new issue page (#14194) (#14207)
+  * Check for notExist on profile repository page (#14197) (#14203)
+
+## [1.13.1](https://github.com/go-gitea/gitea/releases/tag/v1.13.1) - 2020-12-29
+
+* SECURITY
+  * Hide private participation in Orgs (#13994) (#14031)
+  * Fix escaping issue in diff (#14153) (#14154)
+* BUGFIXES
+  * Fix bug of link query order on markdown render (#14156) (#14171)
+  * Drop long repo topics during migration (#14152) (#14155)
+  * Ensure that search term and page are not lost on adoption page-turn (#14133) (#14143)
+  * Fix storage config implementation (#14091) (#14095)
+  * Fix panic in BasicAuthDecode (#14046) (#14048)
+  * Always wait for the cmd to finish (#14006) (#14039)
+  * Don't use simpleMDE editor on mobile devices for 1.13 (#14029)
+  * Fix incorrect review comment diffs (#14002) (#14011)
+  * Trim the branch prefix from action.GetBranch (#13981) (#13986)
+  * Ensure template renderer is available before storage handler (#13164) (#13982)
+  * Whenever the password is updated ensure that the hash algorithm is too (#13966) (#13967)
+  * Enforce setting HEAD in wiki to master (#13950) (#13961)
+  * Fix feishu webhook caused by API changed (#13938)
+  * Fix Quote Reply button on review diff (#13830) (#13898)
+  * Fix Pull Merge when tag with same name as base branch exist (#13882) (#13896)
+  * Fix mermaid chart size (#13865)
+  * Fix branch/tag notifications in mirror sync (#13855) (#13862)
+  * Fix crash in short link processor (#13839) (#13841)
+  * Update font stack to bootstrap's latest (#13834) (#13837)
+  * Make sure email recipients can see issue (#13820) (#13827)
+  * Reply button is not removed when deleting a code review comment (#13824)
+  * When reinitialising DBConfig reset the database use flags (#13796) (#13811)
+* ENHANCEMENTS
+  * Add emoji in label to project boards (#13978) (#14021)
+  * Send webhook when tag is removed via Web UI (#14015) (#14019)
+  * Use Process Manager to create own Context (#13792) (#13793)
+* API
+  * GetCombinedCommitStatusByRef always return json & swagger doc fixes (#14047)
+  * Return original URL of Repositories (#13885) (#13886)
+
+## [1.13.0](https://github.com/go-gitea/gitea/releases/tag/v1.13.0) - 2020-12-01
+* SECURITY
+  * Add Allow-/Block-List for Migrate & Mirrors (#13610) (#13776)
+  * Prevent git operations for inactive users (#13527) (#13536)
+  * Disallow urlencoded new lines in git protocol paths if there is a port (#13521) (#13524)
   * Mitigate Security vulnerability in the git hook feature (#13058)
   * Disable DSA ssh keys by default (#13056)
   * Set TLS minimum version to 1.2 (#12689)
   * Use argon as default password hash algorithm (#12688)
 * BREAKING
+  * Set RUN_MODE prod by default (#13765) (#13767)
   * Don't replace underscores in auto-generated IDs in goldmark (#12805)
   * Add Primary Key to Topic and RepoTopic tables (#12639)
   * Disable password complexity check default (#12557)
@@ -71,6 +224,40 @@ been added to each release, please refer to the [blog](https://blog.gitea.io).
   * Add endpoint for Branch Creation (#11607)
   * Add pagination headers on endpoints that support total count from database (#11145)
 * BUGFIXES
+  * Fix bogus http requests on diffs (#13760) (#13761)
+  * Show 'owner' tag for real owner (#13689) (#13743)
+  * Validate email before inserting/updating (#13475) (#13666)
+  * Fix issue/pull request list assignee filter (#13647) (#13651)
+  * Gitlab migration support for subdirectories (#13563) (#13591)
+  * Fix logic for preferred license setting (#13550) (#13557)
+  * Add missed sync branch/tag webhook (#13538) (#13556)
+  * Migration won't fail on non-migrated reactions (#13507)
+  * Fix Italian language file parsing error (#13156)
+  * Show outdated comments in pull request (#13148) (#13162)
+  * Fix parsing of pre-release git version (#13169) (#13172)
+  * Fix diff skipping lines (#13154) (#13155)
+  * When handling errors in storageHandler check underlying error (#13178) (#13193)
+  * Fix size and clickable area on file table back link (#13205) (#13207)
+  * Add better error checking for inline html diff code (#13251)
+  * Fix initial commit page & binary munching problem (#13249) (#13258)
+  * Fix migrations from remote Gitea instances when configuration not set (#13229) (#13273)
+  * Store task errors following migrations and display them (#13246) (#13287)
+  * Fix bug isEnd detection on getIssues/getPullRequests (#13299) (#13301)
+  * When the git ref is unable to be found return broken pr (#13218) (#13303)
+  * Ensure topics added using the API are added to the repository (#13285) (#13302)
+  * Fix avatar autogeneration (#13233) (#13282)
+  * Add migrated pulls to pull request task queue (#13331) (#13334)
+  * Issue comment reactions should also check pull type on API (#13349) (#13350)
+  * Fix links to repositories in /user/setting/repos (#13360) (#13362)
+  * Remove obsolete change of email on profile page (#13341) (#13347)
+  * Fix scrolling to resolved comment anchors (#13343) (#13371)
+  * Storage configuration support `[storage]` (#13314) (#13379)
+  * When creating line diffs do not split within an html entity (#13357) (#13375) (#13425) (#13427)
+  * Fix reactions on code comments (#13390) (#13401)
+  * Add missing full names when DEFAULT_SHOW_FULL_NAME is enabled (#13424)
+  * Replies to outdated code comments should also be outdated (#13217) (#13433)
+  * Fix panic bug in handling multiple references in commit (#13486) (#13487)
+  * Prevent panic on git blame by limiting lines to 4096 bytes at most (#13470) (#13491)
   * Show original author's reviews on pull summary box (#13127)
   * Update golangci-lint to version 1.31.0 (#13102)
   * Fix line break for MS teams webhook (#13081)
@@ -140,6 +327,10 @@ been added to each release, please refer to the [blog](https://blog.gitea.io).
   * Fix Enter not working in SimpleMDE (#11564)
   * Fix bug about can't skip commits base on base branch (#11555)
 * ENHANCEMENTS
+  * Only Return JSON for responses (#13511) (#13565)
+  * Use existing analyzer module for language detection for highlighting (#13522) (#13551)
+  * Return the full rejection message and errors in flash errors (#13221) (#13237)
+  * Remove PAM from auth dropdown when unavailable (#13276) (#13281)
   * Add HostCertificate to sshd_config in Docker image (#13143)
   * Save TimeStamps for Star, Label, Follow, Watch and Collaboration to Database (#13124)
   * Improve error feedback for duplicate deploy keys (#13112)

Makefile

@@ -585,7 +585,7 @@ release-darwin: | $(DIST_DIRS)
 	@hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
 		GO111MODULE=off $(GO) get -u src.techknowlogick.com/xgo; \
 	fi
-	CGO_CFLAGS="$(CGO_CFLAGS)" GO111MODULE=off xgo -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '$(LDFLAGS)' -targets 'darwin/*' -out gitea-$(VERSION) .
+	CGO_CFLAGS="$(CGO_CFLAGS)" GO111MODULE=off xgo -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '$(LDFLAGS)' -targets 'darwin/amd64' -out gitea-$(VERSION) .
 ifeq ($(CI),drone)
 	cp /build/* $(DIST)/binaries
 endif
@@ -638,8 +638,8 @@ fomantic: $(FOMANTIC_DEST)
 
 $(FOMANTIC_DEST): $(FOMANTIC_CONFIGS) | node_modules
 	rm -rf $(FOMANTIC_DEST_DIR)
-	cp web_src/fomantic/theme.config.less node_modules/fomantic-ui/src/theme.config
-	cp -r web_src/fomantic/_site/* node_modules/fomantic-ui/src/_site/
+	cp -f web_src/fomantic/theme.config.less node_modules/fomantic-ui/src/theme.config
+	cp -fr web_src/fomantic/_site/* node_modules/fomantic-ui/src/_site/
 	npx gulp -f node_modules/fomantic-ui/gulpfile.js build
 	@touch $(FOMANTIC_DEST)
 

cmd/admin.go

@@ -283,7 +283,7 @@ func runChangePassword(c *cli.Context) error {
 	}
 	user.HashPassword(c.String("password"))
 
-	if err := models.UpdateUserCols(user, "passwd", "salt"); err != nil {
+	if err := models.UpdateUserCols(user, "passwd", "passwd_hash_algo", "salt"); err != nil {
 		return err
 	}
 

cmd/doctor.go

@@ -606,6 +606,22 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
 		}
 	}
 
+	// find IssueLabels without existing label
+	count, err = models.CountOrphanedIssueLabels()
+	if err != nil {
+		return nil, err
+	}
+	if count > 0 {
+		if ctx.Bool("fix") {
+			if err = models.DeleteOrphanedIssueLabels(); err != nil {
+				return nil, err
+			}
+			results = append(results, fmt.Sprintf("%d issue_labels without existing label deleted", count))
+		} else {
+			results = append(results, fmt.Sprintf("%d issue_labels without existing label", count))
+		}
+	}
+
 	//find issues without existing repository
 	count, err = models.CountOrphanedIssues()
 	if err != nil {
@@ -670,6 +686,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
 		}
 	}
 
+	if setting.Database.UsePostgreSQL {
+		count, err = models.CountBadSequences()
+		if err != nil {
+			return nil, err
+		}
+		if count > 0 {
+			if ctx.Bool("fix") {
+				err := models.FixBadSequences()
+				if err != nil {
+					return nil, err
+				}
+				results = append(results, fmt.Sprintf("%d sequences updated", count))
+			} else {
+				results = append(results, fmt.Sprintf("%d sequences with incorrect values", count))
+			}
+		}
+	}
 	//ToDo: function to recalc all counters
 
 	return results, nil

custom/conf/app.example.ini

@@ -8,8 +8,8 @@
 APP_NAME = Gitea: Git with a cup of tea
 ; Change it if you run locally
 RUN_USER = git
-; Either "dev", "prod" or "test", default is "dev"
-RUN_MODE = dev
+; Application run mode, affects performance and debugging. Either "dev", "prod" or "test", default is "prod"
+RUN_MODE = prod
 
 [project]
 ; Default templates for project boards
@@ -548,7 +548,7 @@ ONLY_ALLOW_PUSH_IF_GITEA_ENVIRONMENT_SET = true
 ;Classes include "lower,upper,digit,spec"
 PASSWORD_COMPLEXITY = off
 ; Password Hash algorithm, either "argon2", "pbkdf2", "scrypt" or "bcrypt"
-PASSWORD_HASH_ALGO = argon2
+PASSWORD_HASH_ALGO = pbkdf2
 ; Set false to allow JavaScript to read CSRF cookie
 CSRF_COOKIE_HTTP_ONLY = true
 ; Validate against https://haveibeenpwned.com/Passwords to see if a password has been exposed
@@ -850,7 +850,7 @@ MACARON = file
 ROUTER_LOG_LEVEL = Info
 ROUTER = console
 ENABLE_ACCESS_LOG = false
-ACCESS_LOG_TEMPLATE = {{.Ctx.RemoteAddr}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}\" \"{{.Ctx.Req.UserAgent}}"
+ACCESS_LOG_TEMPLATE = {{.Ctx.RemoteAddr}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}\" \"{{.Ctx.Req.UserAgent}}"
 ACCESS = file
 ; Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace"
 LEVEL = Info
@@ -1188,6 +1188,14 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
 MAX_ATTEMPTS = 3
 ; Backoff time per http/https request retry (seconds)
 RETRY_BACKOFF = 3
+; Allowed domains for migrating, default is blank. Blank means everything will be allowed.
+; Multiple domains could be separated by commas.
+ALLOWED_DOMAINS =
+; Blocklist for migrating, default is blank. Multiple domains could be separated by commas.
+; When ALLOWED_DOMAINS is not blank, this option will be ignored.
+BLOCKED_DOMAINS =
+; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default)
+ALLOW_LOCALNETWORKS = false
 
 ; default storage for attachments, lfs and avatars
 [storage]

docker/root/etc/s6/gitea/setup

@@ -25,7 +25,7 @@ if [ ! -f ${GITEA_CUSTOM}/conf/app.ini ]; then
 
     # Substitude the environment variables in the template
     APP_NAME=${APP_NAME:-"Gitea: Git with a cup of tea"} \
-    RUN_MODE=${RUN_MODE:-"dev"} \
+    RUN_MODE=${RUN_MODE:-"prod"} \
     DOMAIN=${DOMAIN:-"localhost"} \
     SSH_DOMAIN=${SSH_DOMAIN:-"localhost"} \
     HTTP_PORT=${HTTP_PORT:-"3000"} \

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -36,9 +36,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `APP_NAME`: **Gitea: Git with a cup of tea**: Application name, used in the page title.
 - `RUN_USER`: **git**: The user Gitea will run as. This should be a dedicated system
    (non-user) account. Setting this incorrectly will cause Gitea to not start.
-- `RUN_MODE`: **dev**: For performance and other purposes, change this to `prod` when
-   deployed to a production environment. The installation process will set this to `prod`
-   automatically. \[prod, dev, test\]
+- `RUN_MODE`: **prod**: Application run mode, affects performance and debugging. Either "dev", "prod" or "test".
 
 ## Repository (`repository`)
 
@@ -278,7 +276,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
 
 - `LFS_START_SERVER`: **false**: Enables git-lfs support.
-- `LFS_CONTENT_PATH`: **%(APP_DATA_PATH)/lfs**:  Default LFS content path. (if it is on local storage.)
+- `LFS_CONTENT_PATH`: **%(APP_DATA_PATH)/lfs**:  DEPRECATED: Default LFS content path. (if it is on local storage.)
 - `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
 - `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
 - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
@@ -404,7 +402,7 @@ relation to port exhaustion.
 - `IMPORT_LOCAL_PATHS`: **false**: Set to `false` to prevent all users (including admin) from importing local path on server.
 - `INTERNAL_TOKEN`: **\<random at every install if no uri set\>**: Secret used to validate communication within Gitea binary.
 - `INTERNAL_TOKEN_URI`: **<empty>**: Instead of defining internal token in the configuration, this configuration option can be used to give Gitea a path to a file that contains the internal token (example value: `file:/etc/gitea/internal_token`)
-- `PASSWORD_HASH_ALGO`: **argon2**: The hash algorithm to use \[argon2, pbkdf2, scrypt, bcrypt\].
+- `PASSWORD_HASH_ALGO`: **pbkdf2**: The hash algorithm to use \[argon2, pbkdf2, scrypt, bcrypt\], argon2 will spend more memory than others.
 - `CSRF_COOKIE_HTTP_ONLY`: **true**: Set false to allow JavaScript to read CSRF cookie.
 - `MIN_PASSWORD_LENGTH`: **6**: Minimum password length for new users.
 - `PASSWORD_COMPLEXITY`: **off**: Comma separated list of character classes required to pass minimum complexity. If left empty or no valid values are specified, checking is disabled (off):
@@ -813,6 +811,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 
 - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
 - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
+- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
+- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
+- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
 
 ## Mirror (`mirror`)
 
@@ -827,7 +828,7 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
 
 - `STORAGE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service or other name defined with `[storage.xxx]`
 - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
-- `CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`.
+- `PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`. If not set it fall back to deprecated LFS_CONTENT_PATH value in [server] section.
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -73,6 +73,7 @@ menu:
 
 - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。
 - `LFS_JWT_SECRET`: LFS 认证密钥，改成自己的。
+- `LFS_CONTENT_PATH`: **已废弃**, 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。
 
 ## Database (`database`)
 
@@ -313,6 +314,9 @@ IS_INPUT_FILE = false
 
 - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。
+- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。
+- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。
+- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
 
 ## LFS (`lfs`)
 
@@ -320,7 +324,7 @@ LFS 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配置将从 `[stora
 
 - `STORAGE_TYPE`: **local**: LFS 的存储类型，`local` 将存储到磁盘，`minio` 将存储到 s3 兼容的对象服务。
 - `SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。
-- `CONTENT_PATH`: 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。
+- `PATH`: 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。
 - `MINIO_ENDPOINT`: **localhost:9000**: Minio 地址，仅当 `LFS_STORAGE_TYPE` 为 `minio` 时有效。
 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID，仅当 `LFS_STORAGE_TYPE` 为 `minio` 时有效。
 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `LFS_STORAGE_TYPE` 为 `minio` 时有效。

docs/content/doc/features/webhooks.en-us.md

@@ -30,6 +30,8 @@ All event pushes are POST requests. The methods currently supported are:
 
 ### Event information
 
+**WARNING**: The `secret` field in the payload is deprecated as of Gitea 1.13.0 and will be removed in 1.14.0: https://github.com/go-gitea/gitea/issues/11755
+
 The following is an example of event information that will be sent by Gitea to
 a Payload URL:
 

docs/content/doc/installation/with-docker.en-us.md

@@ -257,7 +257,7 @@ You can configure some of Gitea's settings via environment variables:
 (Default values are provided in **bold**)
 
 * `APP_NAME`: **"Gitea: Git with a cup of tea"**: Application name, used in the page title.
-* `RUN_MODE`: **dev**: For performance and other purposes, change this to `prod` when deployed to a production environment.
+* `RUN_MODE`: **prod**: Application run mode, affects performance and debugging. Either "dev", "prod" or "test".
 * `DOMAIN`: **localhost**: Domain name of this server, used for the displayed http clone URL in Gitea's UI.
 * `SSH_DOMAIN`: **localhost**: Domain name of this server, used for the displayed ssh clone URL in Gitea's UI. If the install page is enabled, SSH Domain Server takes DOMAIN value in the form (which overwrite this setting on save).
 * `SSH_PORT`: **22**: SSH port displayed in clone URL.

go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	code.gitea.io/gitea-vet v0.2.1
-	code.gitea.io/sdk/gitea v0.13.1
+	code.gitea.io/sdk/gitea v0.13.2
 	gitea.com/lunny/levelqueue v0.3.0
 	gitea.com/macaron/binding v0.0.0-20190822013154-a5f53841ed2b
 	gitea.com/macaron/cache v0.0.0-20190822004001-a6e7fee4ee76
@@ -70,7 +70,7 @@ require (
 	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81
 	github.com/mgechev/revive v1.0.3-0.20200921231451-246eac737dc7
 	github.com/mholt/archiver/v3 v3.3.0
-	github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912
+	github.com/microcosm-cc/bluemonday v1.0.6
 	github.com/minio/minio-go/v7 v7.0.4
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc
@@ -99,15 +99,15 @@ require (
 	github.com/urfave/cli v1.20.0
 	github.com/xanzy/go-gitlab v0.37.0
 	github.com/yohcop/openid-go v1.0.0
-	github.com/yuin/goldmark v1.2.1
+	github.com/yuin/goldmark v1.3.3
 	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
 	github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
 	go.jolheiser.com/hcaptcha v0.0.4
 	go.jolheiser.com/pwn v0.0.3
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
-	golang.org/x/net v0.0.0-20200904194848-62affa334b73
+	golang.org/x/crypto v0.0.0-20201217014255-9d1352758620
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20200918174421-af09f7315aff
+	golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44
 	golang.org/x/text v0.3.3
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.0.0-20200921210052-fa0125251cc4
@@ -117,10 +117,10 @@ require (
 	gopkg.in/ini.v1 v1.61.0
 	gopkg.in/ldap.v3 v3.0.2
 	gopkg.in/yaml.v2 v2.3.0
-	mvdan.cc/xurls/v2 v2.1.0
+	mvdan.cc/xurls/v2 v2.2.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.7
 	xorm.io/xorm v1.0.5
 )
 
-replace github.com/hashicorp/go-version => github.com/6543/go-version v1.2.3
+replace github.com/hashicorp/go-version => github.com/6543/go-version v1.2.4

go.sum

@@ -15,8 +15,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 code.gitea.io/gitea-vet v0.2.1 h1:b30by7+3SkmiftK0RjuXqFvZg2q4p68uoPGuxhzBN0s=
 code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE=
-code.gitea.io/sdk/gitea v0.13.1 h1:Y7bpH2iO6Q0KhhMJfjP/LZ0AmiYITeRQlCD8b0oYqhk=
-code.gitea.io/sdk/gitea v0.13.1/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
+code.gitea.io/sdk/gitea v0.13.2 h1:wAnT/J7Z62q3fJXbgnecoaOBh8CM1Qq0/DakWxiv4yA=
+code.gitea.io/sdk/gitea v0.13.2/go.mod h1:lee2y8LeV3kQb2iK+hHlMqoadL4bp27QOkOV/hawLKg=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gitea.com/lunny/levelqueue v0.3.0 h1:MHn1GuSZkxvVEDMyAPqlc7A3cOW+q8RcGhRgH/xtm6I=
 gitea.com/lunny/levelqueue v0.3.0/go.mod h1:HBqmLbz56JWpfEGG0prskAV97ATNRoj5LDmPicD22hU=
@@ -48,8 +48,8 @@ gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 h1:N9QFoeNsUXLhl14m
 gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7/go.mod h1:kgsbFPPS4P+acDYDOPDa3N4IWWOuDJt5/INKRUz7aks=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
-github.com/6543/go-version v1.2.3 h1:uF30BawMhoQLzqBeCwhFcWM6HVxlzMHe/zXbzJeKP+o=
-github.com/6543/go-version v1.2.3/go.mod h1:fcfWh4zkneEgGXe8JJptiGwp8l6JgJJgS7oTw6P83So=
+github.com/6543/go-version v1.2.4 h1:MPsSnqNrM0HwA9tnmWNnsMdQMg4/u4fflARjwomoof4=
+github.com/6543/go-version v1.2.4/go.mod h1:oqFAHCwtLVUTLdhQmVZWYvaHXTdsbB4SY85at64SQEo=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -140,8 +140,6 @@ github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK
 github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU=
-github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
@@ -649,8 +647,8 @@ github.com/mgechev/revive v1.0.3-0.20200921231451-246eac737dc7 h1:ydVkpU/M4/c45y
 github.com/mgechev/revive v1.0.3-0.20200921231451-246eac737dc7/go.mod h1:no/hfevHbndpXR5CaJahkYCfM/FFpmM/dSOwFGU7Z1o=
 github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig=
 github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao=
-github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912 h1:hJde9rA24hlTcAYSwJoXpDUyGtfKQ/jsofw+WaDqGrI=
-github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
+github.com/microcosm-cc/bluemonday v1.0.6 h1:ZOvqHKtnx0fUpnbQm3m3zKFWE+DRC+XB1onh8JoEObE=
+github.com/microcosm-cc/bluemonday v1.0.6/go.mod h1:HOT/6NaBlR0f9XlxD3zolN6Z3N8Lp4pvhp+jLS5ihnI=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -768,6 +766,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
@@ -884,8 +883,9 @@ github.com/yuin/goldmark v1.1.7/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.3 h1:37BdQwPx8VOSic8eDSWee6QL9mRpZRm9VJp/QugNrW0=
+github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 h1:VWSxtAiQNh3zgHJpdpkpVYjTPqRE3P6UZCOPa1nRDio=
 github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691/go.mod h1:YLF3kDffRfUH/bTxOxHhV6lxwIB3Vfj91rEwNMS9MXo=
 github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 h1:gZucqLjL1eDzVWrXj4uiWeMbAopJlBR2mKQAsTGdPwo=
@@ -936,8 +936,9 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 h1:3wPMTskHO3+O6jqTEXyFcsnuxMQOqYSaHsDxcbUXpqA=
+golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -993,8 +994,9 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1050,8 +1052,12 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
-golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -1196,8 +1202,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-mvdan.cc/xurls/v2 v2.1.0 h1:KaMb5GLhlcSX+e+qhbRJODnUUBvlw01jt4yrjFIHAuA=
-mvdan.cc/xurls/v2 v2.1.0/go.mod h1:5GrSd9rOnKOpZaji1OZLYL/yeAAtGDlo/cFe+8K5n8E=
+mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
+mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 h1:mUcz5b3FJbP5Cvdq7Khzn6J9OCUQJaBwgBkCR+MOwSs=
 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY=

integrations/api_admin_test.go

@@ -144,3 +144,22 @@ func TestAPIListUsersNonAdmin(t *testing.T) {
 	req := NewRequestf(t, "GET", "/api/v1/admin/users?token=%s", token)
 	session.MakeRequest(t, req, http.StatusForbidden)
 }
+
+func TestAPICreateUserInvalidEmail(t *testing.T) {
+	defer prepareTestEnv(t)()
+	adminUsername := "user1"
+	session := loginUser(t, adminUsername)
+	token := getTokenForLoggedInUser(t, session)
+	urlStr := fmt.Sprintf("/api/v1/admin/users?token=%s", token)
+	req := NewRequestWithValues(t, "POST", urlStr, map[string]string{
+		"email":                "invalid_email@domain.com\r\n",
+		"full_name":            "invalid user",
+		"login_name":           "invalidUser",
+		"must_change_password": "true",
+		"password":             "password",
+		"send_notify":          "true",
+		"source_id":            "0",
+		"username":             "invalidUser",
+	})
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity)
+}

integrations/api_helper_for_declarative_test.go

@@ -5,14 +5,17 @@
 package integrations
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
+	"code.gitea.io/gitea/modules/queue"
 	api "code.gitea.io/gitea/modules/structs"
 
 	"github.com/stretchr/testify/assert"
@@ -225,11 +228,29 @@ func doAPIMergePullRequest(ctx APITestContext, owner, repo string, index int64)
 			Do:                string(models.MergeStyleMerge),
 		})
 
-		if ctx.ExpectedCode != 0 {
-			ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)
-			return
+		resp := ctx.Session.MakeRequest(t, req, NoExpectedStatus)
+
+		if resp.Code == http.StatusMethodNotAllowed {
+			err := api.APIError{}
+			DecodeJSON(t, resp, &err)
+			assert.EqualValues(t, "Please try again later", err.Message)
+			queue.GetManager().FlushAll(context.Background(), 5*time.Second)
+			req = NewRequestWithJSON(t, http.MethodPost, urlStr, &auth.MergePullRequestForm{
+				MergeMessageField: "doAPIMergePullRequest Merge",
+				Do:                string(models.MergeStyleMerge),
+			})
+			resp = ctx.Session.MakeRequest(t, req, NoExpectedStatus)
+		}
+
+		expected := ctx.ExpectedCode
+		if expected == 0 {
+			expected = 200
+		}
+
+		if !assert.EqualValues(t, expected, resp.Code,
+			"Request: %s %s", req.Method, req.URL.String()) {
+			logUnexpectedResponse(t, resp)
 		}
-		ctx.Session.MakeRequest(t, req, 200)
 	}
 }
 

integrations/api_oauth2_apps_test.go

@@ -92,6 +92,10 @@ func testAPIDeleteOAuth2Application(t *testing.T) {
 	session.MakeRequest(t, req, http.StatusNoContent)
 
 	models.AssertNotExistsBean(t, &models.OAuth2Application{UID: oldApp.UID, Name: oldApp.Name})
+
+	// Delete again will return not found
+	req = NewRequest(t, "DELETE", urlStr)
+	session.MakeRequest(t, req, http.StatusNotFound)
 }
 
 func testAPIGetOAuth2Application(t *testing.T) {

integrations/api_pull_test.go

@@ -74,8 +74,79 @@ func TestAPICreatePullSuccess(t *testing.T) {
 		Base:  "master",
 		Title: "create a failure pr",
 	})
-
 	session.MakeRequest(t, req, 201)
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity) // second request should fail
+}
+
+func TestAPICreatePullWithFieldsSuccess(t *testing.T) {
+	defer prepareTestEnv(t)()
+	// repo10 have code, pulls units.
+	repo10 := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 10}).(*models.Repository)
+	owner10 := models.AssertExistsAndLoadBean(t, &models.User{ID: repo10.OwnerID}).(*models.User)
+	// repo11 only have code unit but should still create pulls
+	repo11 := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 11}).(*models.Repository)
+	owner11 := models.AssertExistsAndLoadBean(t, &models.User{ID: repo11.OwnerID}).(*models.User)
+
+	session := loginUser(t, owner11.Name)
+	token := getTokenForLoggedInUser(t, session)
+
+	opts := &api.CreatePullRequestOption{
+		Head:      fmt.Sprintf("%s:master", owner11.Name),
+		Base:      "master",
+		Title:     "create a failure pr",
+		Body:      "foobaaar",
+		Milestone: 5,
+		Assignees: []string{owner10.Name},
+		Labels:    []int64{5},
+	}
+
+	req := NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls?token=%s", owner10.Name, repo10.Name, token), opts)
+
+	res := session.MakeRequest(t, req, 201)
+	pull := new(api.PullRequest)
+	DecodeJSON(t, res, pull)
+
+	assert.NotNil(t, pull.Milestone)
+	assert.EqualValues(t, opts.Milestone, pull.Milestone.ID)
+	if assert.Len(t, pull.Assignees, 1) {
+		assert.EqualValues(t, opts.Assignees[0], owner10.Name)
+	}
+	assert.NotNil(t, pull.Labels)
+	assert.EqualValues(t, opts.Labels[0], pull.Labels[0].ID)
+}
+
+func TestAPICreatePullWithFieldsFailure(t *testing.T) {
+	defer prepareTestEnv(t)()
+	// repo10 have code, pulls units.
+	repo10 := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 10}).(*models.Repository)
+	owner10 := models.AssertExistsAndLoadBean(t, &models.User{ID: repo10.OwnerID}).(*models.User)
+	// repo11 only have code unit but should still create pulls
+	repo11 := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 11}).(*models.Repository)
+	owner11 := models.AssertExistsAndLoadBean(t, &models.User{ID: repo11.OwnerID}).(*models.User)
+
+	session := loginUser(t, owner11.Name)
+	token := getTokenForLoggedInUser(t, session)
+
+	opts := &api.CreatePullRequestOption{
+		Head: fmt.Sprintf("%s:master", owner11.Name),
+		Base: "master",
+	}
+
+	req := NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls?token=%s", owner10.Name, repo10.Name, token), opts)
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity)
+	opts.Title = "is required"
+
+	opts.Milestone = 666
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity)
+	opts.Milestone = 5
+
+	opts.Assignees = []string{"qweruqweroiuyqweoiruywqer"}
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity)
+	opts.Assignees = []string{owner10.LoginName}
+
+	opts.Labels = []int64{55555}
+	session.MakeRequest(t, req, http.StatusUnprocessableEntity)
+	opts.Labels = []int64{5}
 }
 
 func TestAPIEditPull(t *testing.T) {

integrations/api_repo_test.go

@@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) {
 		{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden},
 		{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated},
 		{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden},
+		{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity},
+		{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity},
 	}
 
 	defer prepareTestEnv(t)()
@@ -325,8 +327,16 @@ func TestAPIRepoMigrate(t *testing.T) {
 		if resp.Code == http.StatusUnprocessableEntity {
 			respJSON := map[string]string{}
 			DecodeJSON(t, resp, &respJSON)
-			if assert.Equal(t, respJSON["message"], "Remote visit addressed rate limitation.") {
+			switch respJSON["message"] {
+			case "Remote visit addressed rate limitation.":
 				t.Log("test hit github rate limitation")
+			case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'":
+				assert.EqualValues(t, "private-ip", testCase.repoName)
+			case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'",
+				"migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'":
+				assert.EqualValues(t, "local-ip", testCase.repoName)
+			default:
+				t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL)
 			}
 		} else {
 			assert.EqualValues(t, testCase.expectedStatus, resp.Code)
@@ -435,11 +445,12 @@ func TestAPIRepoTransfer(t *testing.T) {
 		expectedStatus int
 	}{
 		{ctxUserID: 1, newOwner: "user2", teams: nil, expectedStatus: http.StatusAccepted},
-		{ctxUserID: 2, newOwner: "user1", teams: nil, expectedStatus: http.StatusAccepted},
+		{ctxUserID: 2, newOwner: "user1", teams: nil, expectedStatus: http.StatusForbidden},
 		{ctxUserID: 2, newOwner: "user6", teams: nil, expectedStatus: http.StatusForbidden},
 		{ctxUserID: 1, newOwner: "user2", teams: &[]int64{2}, expectedStatus: http.StatusUnprocessableEntity},
 		{ctxUserID: 1, newOwner: "user3", teams: &[]int64{5}, expectedStatus: http.StatusForbidden},
 		{ctxUserID: 1, newOwner: "user3", teams: &[]int64{2}, expectedStatus: http.StatusAccepted},
+		{ctxUserID: 2, newOwner: "user2", teams: nil, expectedStatus: http.StatusAccepted},
 	}
 
 	defer prepareTestEnv(t)()

integrations/api_user_heatmap_test.go

@@ -26,7 +26,7 @@ func TestUserHeatmap(t *testing.T) {
 	var heatmap []*models.UserHeatmapData
 	DecodeJSON(t, resp, &heatmap)
 	var dummyheatmap []*models.UserHeatmapData
-	dummyheatmap = append(dummyheatmap, &models.UserHeatmapData{Timestamp: 1571616000, Contributions: 1})
+	dummyheatmap = append(dummyheatmap, &models.UserHeatmapData{Timestamp: 1603152000, Contributions: 1})
 
 	assert.Equal(t, dummyheatmap, heatmap)
 }

integrations/attachment_test.go

@@ -122,7 +122,7 @@ func TestGetAttachment(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			//Write empty file to be available for response
 			if tc.createFile {
-				_, err := storage.Attachments.Save(models.AttachmentRelativePath(tc.uuid), strings.NewReader("hello world"))
+				_, err := storage.Attachments.Save(models.AttachmentRelativePath(tc.uuid), strings.NewReader("hello world"), -1)
 				assert.NoError(t, err)
 			}
 			//Actual test

integrations/auth_ldap_test.go

@@ -141,7 +141,7 @@ func TestLDAPUserSignin(t *testing.T) {
 
 	assert.Equal(t, u.UserName, htmlDoc.GetInputValueByName("name"))
 	assert.Equal(t, u.FullName, htmlDoc.GetInputValueByName("full_name"))
-	assert.Equal(t, u.Email, htmlDoc.GetInputValueByName("email"))
+	assert.Equal(t, u.Email, htmlDoc.Find(`label[for="email"]`).Siblings().First().Text())
 }
 
 func TestLDAPUserSync(t *testing.T) {
@@ -237,6 +237,6 @@ func TestLDAPUserSSHKeySync(t *testing.T) {
 			syncedKeys[i] = strings.TrimSpace(divs.Eq(i).Text())
 		}
 
-		assert.ElementsMatch(t, u.SSHKeys, syncedKeys)
+		assert.ElementsMatch(t, u.SSHKeys, syncedKeys, "Unequal number of keys synchronized for user: %s", u.UserName)
 	}
 }

integrations/git_helper_for_declarative_test.go

@@ -111,7 +111,7 @@ func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL), prepare ...bo
 
 func doGitClone(dstLocalPath string, u *url.URL) func(*testing.T) {
 	return func(t *testing.T) {
-		assert.NoError(t, git.CloneWithArgs(u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{}))
+		assert.NoError(t, git.CloneWithArgs(context.Background(), u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{}))
 		assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md")))
 	}
 }

integrations/html_helper.go

@@ -37,6 +37,13 @@ func (doc *HTMLDoc) GetInputValueByName(name string) string {
 	return text
 }
 
+// Find gets the descendants of each element in the current set of
+// matched elements, filtered by a selector. It returns a new Selection
+// object containing these matched elements.
+func (doc *HTMLDoc) Find(selector string) *goquery.Selection {
+	return doc.doc.Find(selector)
+}
+
 // GetCSRF for get CSRC token value from input
 func (doc *HTMLDoc) GetCSRF() string {
 	return doc.GetInputValueByName("_csrf")

integrations/integration_test.go

@@ -11,7 +11,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/http/cookiejar"
 	"net/http/httptest"
@@ -27,8 +26,10 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/routers/routes"
@@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
 }
 
 func TestMain(m *testing.M) {
+	defer log.Close()
+
 	managerCtx, cancel := context.WithCancel(context.Background())
 	graceful.InitManager(managerCtx)
 	defer cancel()
@@ -142,6 +145,10 @@ func initIntegrationTest() {
 	util.RemoveAll(models.LocalCopyPath())
 	setting.CheckLFSVersion()
 	setting.InitDBConfig()
+	if err := storage.Init(); err != nil {
+		fmt.Printf("Init storage failed: %v", err)
+		os.Exit(1)
+	}
 
 	switch {
 	case setting.Database.UseMySQL:
@@ -149,27 +156,27 @@ func initIntegrationTest() {
 			setting.Database.User, setting.Database.Passwd, setting.Database.Host))
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
-			log.Fatalf("db.Exec: %v", err)
+			log.Fatal("db.Exec: %v", err)
 		}
 	case setting.Database.UsePostgreSQL:
 		db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
 			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
 		if err != nil {
-			log.Fatalf("db.Query: %v", err)
+			log.Fatal("db.Query: %v", err)
 		}
 		defer dbrows.Close()
 
 		if !dbrows.Next() {
 			if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
-				log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
+				log.Fatal("db.Exec: CREATE DATABASE: %v", err)
 			}
 		}
 		// Check if we need to setup a specific schema
@@ -183,18 +190,18 @@ func initIntegrationTest() {
 		// This is a different db object; requires a different Close()
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
 		if err != nil {
-			log.Fatalf("db.Query: %v", err)
+			log.Fatal("db.Query: %v", err)
 		}
 		defer schrows.Close()
 
 		if !schrows.Next() {
 			// Create and setup a DB schema
 			if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
-				log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
+				log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
 			}
 		}
 
@@ -203,10 +210,10 @@ func initIntegrationTest() {
 		db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
 			host, port, "master", setting.Database.User, setting.Database.Passwd))
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
-			log.Fatalf("db.Exec: %v", err)
+			log.Fatal("db.Exec: %v", err)
 		}
 		defer db.Close()
 	}

integrations/lfs_getobject_test.go

@@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
 			}
 		}
 	}
+
 	resp := session.MakeRequest(t, req, expectedStatus)
 
 	return resp
@@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
 		{"bytes=0-10", "123456789\n", http.StatusPartialContent},
 		// end-range bigger than length-1 is ignored
 		{"bytes=0-11", "123456789\n", http.StatusPartialContent},
-		{"bytes=11-", "", http.StatusPartialContent},
+		{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
 		// incorrect header value cause whole header to be ignored
 		{"bytes=-", "123456789\n", http.StatusOK},
 		{"foobar", "123456789\n", http.StatusOK},

integrations/mysql.ini.tmpl

@@ -45,19 +45,21 @@ START_SSH_SERVER = true
 OFFLINE_MODE     = false
 
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
-LFS_STORE_TYPE = minio
-LFS_SERVE_DIRECT = false
-LFS_MINIO_ENDPOINT = minio:9000
-LFS_MINIO_ACCESS_KEY_ID = 123456
-LFS_MINIO_SECRET_ACCESS_KEY = 12345678
-LFS_MINIO_BUCKET = gitea
-LFS_MINIO_LOCATION = us-east-1
-LFS_MINIO_BASE_PATH = lfs/
-LFS_MINIO_USE_SSL = false
+
+[lfs]
+MINIO_BASE_PATH = lfs/
 
 [attachment]
+MINIO_BASE_PATH = attachments/
+
+[avatars]
+MINIO_BASE_PATH = avatars/
+
+[repo-avatars]
+MINIO_BASE_PATH = repo-avatars/
+
+[storage]
 STORAGE_TYPE = minio
 SERVE_DIRECT = false
 MINIO_ENDPOINT = minio:9000
@@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456
 MINIO_SECRET_ACCESS_KEY = 12345678
 MINIO_BUCKET = gitea
 MINIO_LOCATION = us-east-1
-MINIO_BASE_PATH = attachments/
 MINIO_USE_SSL = false
 
 [mailer]
@@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL                = true
 DISABLE_GRAVATAR              = false
 ENABLE_FEDERATED_AVATAR       = false
 
-AVATAR_UPLOAD_PATH            = integrations/gitea-integration-mysql/data/avatars
-REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
-
 [session]
 PROVIDER        = file
 PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions

integrations/signup_test.go

@@ -5,10 +5,14 @@
 package integrations
 
 import (
+	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 
 	"code.gitea.io/gitea/modules/setting"
+	"github.com/stretchr/testify/assert"
+	"github.com/unknwon/i18n"
 )
 
 func TestSignup(t *testing.T) {
@@ -28,3 +32,37 @@ func TestSignup(t *testing.T) {
 	req = NewRequest(t, "GET", "/exampleUser")
 	MakeRequest(t, req, http.StatusOK)
 }
+
+func TestSignupEmail(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	setting.Service.EnableCaptcha = false
+
+	tests := []struct {
+		email      string
+		wantStatus int
+		wantMsg    string
+	}{
+		{"exampleUser@example.com\r\n", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)},
+		{"exampleUser@example.com\r", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)},
+		{"exampleUser@example.com\n", http.StatusOK, i18n.Tr("en", "form.email_invalid", nil)},
+		{"exampleUser@example.com", http.StatusFound, ""},
+	}
+
+	for i, test := range tests {
+		req := NewRequestWithValues(t, "POST", "/user/sign_up", map[string]string{
+			"user_name": fmt.Sprintf("exampleUser%d", i),
+			"email":     test.email,
+			"password":  "examplePassword!1",
+			"retype":    "examplePassword!1",
+		})
+		resp := MakeRequest(t, req, test.wantStatus)
+		if test.wantMsg != "" {
+			htmlDoc := NewHTMLParser(t, resp.Body)
+			assert.Equal(t,
+				test.wantMsg,
+				strings.TrimSpace(htmlDoc.doc.Find(".ui.message").Text()),
+			)
+		}
+	}
+}

models/action.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -243,7 +244,7 @@ func (a *Action) getCommentLink(e Engine) string {
 
 // GetBranch returns the action's repository branch.
 func (a *Action) GetBranch() string {
-	return a.RefName
+	return strings.TrimPrefix(a.RefName, git.BranchPrefix)
 }
 
 // GetContent returns the action's content.

models/admin.go

@@ -77,7 +77,7 @@ func removeStorageWithNotice(e Engine, bucket storage.ObjectStorage, title, path
 	if err := bucket.Delete(path); err != nil {
 		desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
 		log.Warn(title+" [%s]: %v", path, err)
-		if err = createNotice(x, NoticeRepository, desc); err != nil {
+		if err = createNotice(e, NoticeRepository, desc); err != nil {
 			log.Error("CreateRepositoryNotice: %v", err)
 		}
 	}

models/attachment.go

@@ -99,7 +99,7 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
 func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
 	attach.UUID = gouuid.New().String()
 
-	size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file))
+	size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file), -1)
 	if err != nil {
 		return nil, fmt.Errorf("Create: %v", err)
 	}

models/commit_status_test.go

@@ -18,7 +18,7 @@ func TestGetCommitStatuses(t *testing.T) {
 
 	sha1 := "1234123412341234123412341234123412341234"
 
-	statuses, maxResults, err := GetCommitStatuses(repo1, sha1, &CommitStatusOptions{})
+	statuses, maxResults, err := GetCommitStatuses(repo1, sha1, &CommitStatusOptions{ListOptions: ListOptions{Page: 1, PageSize: 50}})
 	assert.NoError(t, err)
 	assert.Equal(t, int(maxResults), 5)
 	assert.Len(t, statuses, 5)

models/consistency.go

@@ -5,10 +5,13 @@
 package models
 
 import (
+	"fmt"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/setting"
 	"github.com/stretchr/testify/assert"
 	"xorm.io/builder"
 )
@@ -221,6 +224,24 @@ func DeleteOrphanedLabels() error {
 	return nil
 }
 
+// CountOrphanedIssueLabels return count of IssueLabels witch have no label behind anymore
+func CountOrphanedIssueLabels() (int64, error) {
+	return x.Table("issue_label").
+		Join("LEFT", "label", "issue_label.label_id = label.id").
+		Where(builder.IsNull{"label.id"}).Count()
+}
+
+// DeleteOrphanedIssueLabels delete IssueLabels witch have no label behind anymore
+func DeleteOrphanedIssueLabels() error {
+
+	_, err := x.In("id", builder.Select("issue_label.id").From("issue_label").
+		Join("LEFT", "label", "issue_label.label_id = label.id").
+		Where(builder.IsNull{"label.id"})).
+		Delete(IssueLabel{})
+
+	return err
+}
+
 // CountOrphanedIssues count issues without a repo
 func CountOrphanedIssues() (int64, error) {
 	return x.Table("issue").
@@ -276,11 +297,15 @@ func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) {
 
 // DeleteOrphanedObjects delete subjects with have no existing refobject anymore
 func DeleteOrphanedObjects(subject, refobject, joinCond string) error {
-	_, err := x.In("id", builder.Select("`"+subject+"`.id").
+	subQuery := builder.Select("`"+subject+"`.id").
 		From("`"+subject+"`").
 		Join("LEFT", "`"+refobject+"`", joinCond).
-		Where(builder.IsNull{"`" + refobject + "`.id"})).
-		Delete("`" + subject + "`")
+		Where(builder.IsNull{"`" + refobject + "`.id"})
+	sql, args, err := builder.Delete(builder.In("id", subQuery)).From("`" + subject + "`").ToSQL()
+	if err != nil {
+		return err
+	}
+	_, err = x.Exec(append([]interface{}{sql}, args...)...)
 	return err
 }
 
@@ -295,3 +320,61 @@ func FixNullArchivedRepository() (int64, error) {
 		IsArchived: false,
 	})
 }
+
+// CountBadSequences looks for broken sequences from recreate-table mistakes
+func CountBadSequences() (int64, error) {
+	if !setting.Database.UsePostgreSQL {
+		return 0, nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	var sequences []string
+	schema := sess.Engine().Dialect().URI().Schema
+
+	sess.Engine().SetSchema("")
+	if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+		return 0, err
+	}
+	sess.Engine().SetSchema(schema)
+
+	return int64(len(sequences)), nil
+}
+
+// FixBadSequences fixes for broken sequences from recreate-table mistakes
+func FixBadSequences() error {
+	if !setting.Database.UsePostgreSQL {
+		return nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	var sequences []string
+	schema := sess.Engine().Dialect().URI().Schema
+
+	sess.Engine().SetSchema("")
+	if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+		return err
+	}
+	sess.Engine().SetSchema(schema)
+
+	sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
+
+	for _, sequence := range sequences {
+		tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
+		newSequenceName := tableName + "_id_seq"
+		if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+			return err
+		}
+		if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+			return err
+		}
+	}
+
+	return sess.Commit()
+}

models/consistency_test.go

@@ -0,0 +1,32 @@
+// Copyright 2021 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDeleteOrphanedObjects(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	countBefore, err := x.Count(&PullRequest{})
+	assert.NoError(t, err)
+
+	_, err = x.Insert(&PullRequest{IssueID: 1000}, &PullRequest{IssueID: 1001}, &PullRequest{IssueID: 1003})
+	assert.NoError(t, err)
+
+	orphaned, err := CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
+	assert.NoError(t, err)
+	assert.EqualValues(t, 3, orphaned)
+
+	err = DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
+	assert.NoError(t, err)
+
+	countAfter, err := x.Count(&PullRequest{})
+	assert.NoError(t, err)
+	assert.EqualValues(t, countBefore, countAfter)
+}

models/error.go

@@ -193,6 +193,21 @@ func (err ErrEmailAlreadyUsed) Error() string {
 	return fmt.Sprintf("e-mail already in use [email: %s]", err.Email)
 }
 
+// ErrEmailInvalid represents an error where the email address does not comply with RFC 5322
+type ErrEmailInvalid struct {
+	Email string
+}
+
+// IsErrEmailInvalid checks if an error is an ErrEmailInvalid
+func IsErrEmailInvalid(err error) bool {
+	_, ok := err.(ErrEmailInvalid)
+	return ok
+}
+
+func (err ErrEmailInvalid) Error() string {
+	return fmt.Sprintf("e-mail invalid [email: %s]", err.Email)
+}
+
 // ErrOpenIDAlreadyUsed represents a "OpenIDAlreadyUsed" kind of error.
 type ErrOpenIDAlreadyUsed struct {
 	OpenID string
@@ -1004,6 +1019,29 @@ func IsErrWontSign(err error) bool {
 	return ok
 }
 
+// ErrMigrationNotAllowed explains why a migration from an url is not allowed
+type ErrMigrationNotAllowed struct {
+	Host          string
+	NotResolvedIP bool
+	PrivateNet    string
+}
+
+func (e *ErrMigrationNotAllowed) Error() string {
+	if e.NotResolvedIP {
+		return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host)
+	}
+	if len(e.PrivateNet) != 0 {
+		return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet)
+	}
+	return fmt.Sprintf("migrate from '%s is not allowed'", e.Host)
+}
+
+// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed
+func IsErrMigrationNotAllowed(err error) bool {
+	_, ok := err.(*ErrMigrationNotAllowed)
+	return ok
+}
+
 // __________                             .__
 // \______   \____________    ____   ____ |  |__
 //  |    |  _/\_  __ \__  \  /    \_/ ___\|  |  \
@@ -2003,7 +2041,7 @@ type ErrNotValidReviewRequest struct {
 
 // IsErrNotValidReviewRequest checks if an error is a ErrNotValidReviewRequest.
 func IsErrNotValidReviewRequest(err error) bool {
-	_, ok := err.(ErrReviewNotExist)
+	_, ok := err.(ErrNotValidReviewRequest)
 	return ok
 }
 

models/fixtures/action.yml

@@ -5,7 +5,7 @@
   act_user_id: 2
   repo_id: 2
   is_private: true
-  created_unix: 1571686356
+  created_unix: 1603228283
 
 -
   id: 2

models/fixtures/label.yml

@@ -33,3 +33,11 @@
   num_issues: 1
   num_closed_issues: 0
   
+-
+  id: 5
+  repo_id: 10
+  org_id: 0
+  name: pull-test-label
+  color: '#000000'
+  num_issues: 0
+  num_closed_issues: 0

models/fixtures/milestone.yml

@@ -29,3 +29,11 @@
   content: content random
   is_closed: false
   num_issues: 0
+
+- 
+  id: 5
+  repo_id: 10
+  name: milestone of repo 10 
+  content: for testing with PRs
+  is_closed: false
+  num_issues: 0

models/fixtures/repository.yml

@@ -146,6 +146,7 @@
   num_closed_issues: 0
   num_pulls: 1
   num_closed_pulls: 0
+  num_milestones: 1
   is_mirror: false
   num_forks: 1
   status: 0

models/gpg_key.go

@@ -65,7 +65,11 @@ func (key *GPGKey) AfterLoad(session *xorm.Session) {
 
 // ListGPGKeys returns a list of public keys belongs to given user.
 func ListGPGKeys(uid int64, listOptions ListOptions) ([]*GPGKey, error) {
-	sess := x.Where("owner_id=? AND primary_key_id=''", uid)
+	return listGPGKeys(x, uid, listOptions)
+}
+
+func listGPGKeys(e Engine, uid int64, listOptions ListOptions) ([]*GPGKey, error) {
+	sess := e.Table(&GPGKey{}).Where("owner_id=? AND primary_key_id=''", uid)
 	if listOptions.Page != 0 {
 		sess = listOptions.setSessionPagination(sess)
 	}

models/issue.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
@@ -1491,6 +1492,7 @@ type UserIssueStatsOptions struct {
 	IsPull      bool
 	IsClosed    bool
 	IssueIDs    []int64
+	LabelIDs    []int64
 }
 
 // GetUserIssueStats returns issue statistic information for dashboard by given conditions.
@@ -1507,29 +1509,38 @@ func GetUserIssueStats(opts UserIssueStatsOptions) (*IssueStats, error) {
 		cond = cond.And(builder.In("issue.id", opts.IssueIDs))
 	}
 
+	sess := func(cond builder.Cond) *xorm.Session {
+		s := x.Where(cond)
+		if len(opts.LabelIDs) > 0 {
+			s.Join("INNER", "issue_label", "issue_label.issue_id = issue.id").
+				In("issue_label.label_id", opts.LabelIDs)
+		}
+		return s
+	}
+
 	switch opts.FilterMode {
 	case FilterModeAll:
-		stats.OpenCount, err = x.Where(cond).And("issue.is_closed = ?", false).
+		stats.OpenCount, err = sess(cond).And("issue.is_closed = ?", false).
 			And(builder.In("issue.repo_id", opts.UserRepoIDs)).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
-		stats.ClosedCount, err = x.Where(cond).And("issue.is_closed = ?", true).
+		stats.ClosedCount, err = sess(cond).And("issue.is_closed = ?", true).
 			And(builder.In("issue.repo_id", opts.UserRepoIDs)).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
 	case FilterModeAssign:
-		stats.OpenCount, err = x.Where(cond).And("issue.is_closed = ?", false).
+		stats.OpenCount, err = sess(cond).And("issue.is_closed = ?", false).
 			Join("INNER", "issue_assignees", "issue.id = issue_assignees.issue_id").
 			And("issue_assignees.assignee_id = ?", opts.UserID).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
-		stats.ClosedCount, err = x.Where(cond).And("issue.is_closed = ?", true).
+		stats.ClosedCount, err = sess(cond).And("issue.is_closed = ?", true).
 			Join("INNER", "issue_assignees", "issue.id = issue_assignees.issue_id").
 			And("issue_assignees.assignee_id = ?", opts.UserID).
 			Count(new(Issue))
@@ -1537,27 +1548,27 @@ func GetUserIssueStats(opts UserIssueStatsOptions) (*IssueStats, error) {
 			return nil, err
 		}
 	case FilterModeCreate:
-		stats.OpenCount, err = x.Where(cond).And("issue.is_closed = ?", false).
+		stats.OpenCount, err = sess(cond).And("issue.is_closed = ?", false).
 			And("issue.poster_id = ?", opts.UserID).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
-		stats.ClosedCount, err = x.Where(cond).And("issue.is_closed = ?", true).
+		stats.ClosedCount, err = sess(cond).And("issue.is_closed = ?", true).
 			And("issue.poster_id = ?", opts.UserID).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
 	case FilterModeMention:
-		stats.OpenCount, err = x.Where(cond).And("issue.is_closed = ?", false).
+		stats.OpenCount, err = sess(cond).And("issue.is_closed = ?", false).
 			Join("INNER", "issue_user", "issue.id = issue_user.issue_id and issue_user.is_mentioned = ?", true).
 			And("issue_user.uid = ?", opts.UserID).
 			Count(new(Issue))
 		if err != nil {
 			return nil, err
 		}
-		stats.ClosedCount, err = x.Where(cond).And("issue.is_closed = ?", true).
+		stats.ClosedCount, err = sess(cond).And("issue.is_closed = ?", true).
 			Join("INNER", "issue_user", "issue.id = issue_user.issue_id and issue_user.is_mentioned = ?", true).
 			And("issue_user.uid = ?", opts.UserID).
 			Count(new(Issue))
@@ -1567,7 +1578,7 @@ func GetUserIssueStats(opts UserIssueStatsOptions) (*IssueStats, error) {
 	}
 
 	cond = cond.And(builder.Eq{"issue.is_closed": opts.IsClosed})
-	stats.AssignCount, err = x.Where(cond).
+	stats.AssignCount, err = sess(cond).
 		Join("INNER", "issue_assignees", "issue.id = issue_assignees.issue_id").
 		And("issue_assignees.assignee_id = ?", opts.UserID).
 		Count(new(Issue))
@@ -1575,14 +1586,14 @@ func GetUserIssueStats(opts UserIssueStatsOptions) (*IssueStats, error) {
 		return nil, err
 	}
 
-	stats.CreateCount, err = x.Where(cond).
+	stats.CreateCount, err = sess(cond).
 		And("poster_id = ?", opts.UserID).
 		Count(new(Issue))
 	if err != nil {
 		return nil, err
 	}
 
-	stats.MentionCount, err = x.Where(cond).
+	stats.MentionCount, err = sess(cond).
 		Join("INNER", "issue_user", "issue.id = issue_user.issue_id and issue_user.is_mentioned = ?", true).
 		And("issue_user.uid = ?", opts.UserID).
 		Count(new(Issue))
@@ -1590,7 +1601,7 @@ func GetUserIssueStats(opts UserIssueStatsOptions) (*IssueStats, error) {
 		return nil, err
 	}
 
-	stats.YourRepositoriesCount, err = x.Where(cond).
+	stats.YourRepositoriesCount, err = sess(cond).
 		And(builder.In("issue.repo_id", opts.UserRepoIDs)).
 		Count(new(Issue))
 	if err != nil {
@@ -1829,6 +1840,19 @@ func (issue *Issue) updateClosedNum(e Engine) (err error) {
 	return
 }
 
+// FindAndUpdateIssueMentions finds users mentioned in the given content string, and saves them in the database.
+func (issue *Issue) FindAndUpdateIssueMentions(ctx DBContext, doer *User, content string) (mentions []*User, err error) {
+	rawMentions := references.FindAllMentionsMarkdown(content)
+	mentions, err = issue.ResolveMentionsByVisibility(ctx, doer, rawMentions)
+	if err != nil {
+		return nil, fmt.Errorf("UpdateIssueMentions [%d]: %v", issue.ID, err)
+	}
+	if err = UpdateIssueMentions(ctx, issue.ID, mentions); err != nil {
+		return nil, fmt.Errorf("UpdateIssueMentions [%d]: %v", issue.ID, err)
+	}
+	return
+}
+
 // ResolveMentionsByVisibility returns the users mentioned in an issue, removing those that
 // don't have access to reading it. Teams are expanded into their users, but organizations are ignored.
 func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, mentions []string) (users []*User, err error) {

models/issue_assignees.go

@@ -82,7 +82,7 @@ func isUserAssignedToIssue(e Engine, issue *Issue, user *User) (isAssigned bool,
 }
 
 // ClearAssigneeByUserID deletes all assignments of an user
-func clearAssigneeByUserID(sess *xorm.Session, userID int64) (err error) {
+func clearAssigneeByUserID(sess Engine, userID int64) (err error) {
 	_, err = sess.Delete(&IssueAssignees{AssigneeID: userID})
 	return
 }

models/issue_comment.go

@@ -725,6 +725,7 @@ func createComment(e *xorm.Session, opts *CreateCommentOptions) (_ *Comment, err
 		RefAction:        opts.RefAction,
 		RefIsPull:        opts.RefIsPull,
 		IsForcePush:      opts.IsForcePush,
+		Invalidated:      opts.Invalidated,
 	}
 	if _, err = e.Insert(comment); err != nil {
 		return nil, err
@@ -891,6 +892,7 @@ type CreateCommentOptions struct {
 	RefAction        references.XRefAction
 	RefIsPull        bool
 	IsForcePush      bool
+	Invalidated      bool
 }
 
 // CreateComment creates comment of issue or commit.
@@ -966,6 +968,8 @@ type FindCommentsOptions struct {
 	ReviewID int64
 	Since    int64
 	Before   int64
+	Line     int64
+	TreePath string
 	Type     CommentType
 }
 
@@ -989,6 +993,12 @@ func (opts *FindCommentsOptions) toConds() builder.Cond {
 	if opts.Type != CommentTypeUnknown {
 		cond = cond.And(builder.Eq{"comment.type": opts.Type})
 	}
+	if opts.Line > 0 {
+		cond = cond.And(builder.Eq{"comment.line": opts.Line})
+	}
+	if len(opts.TreePath) > 0 {
+		cond = cond.And(builder.Eq{"comment.tree_path": opts.TreePath})
+	}
 	return cond
 }
 
@@ -1003,6 +1013,8 @@ func findComments(e Engine, opts FindCommentsOptions) ([]*Comment, error) {
 		sess = opts.setSessionPagination(sess)
 	}
 
+	// WARNING: If you change this order you will need to fix createCodeComment
+
 	return comments, sess.
 		Asc("comment.created_unix").
 		Asc("comment.id").
@@ -1065,6 +1077,10 @@ func DeleteComment(comment *Comment, doer *User) error {
 		return err
 	}
 
+	if err := deleteReaction(sess, &ReactionOptions{Comment: comment}); err != nil {
+		return err
+	}
+
 	return sess.Commit()
 }
 
@@ -1124,6 +1140,10 @@ func fetchCodeCommentsByReview(e Engine, issue *Issue, currentUser *User, review
 			return nil, err
 		}
 
+		if err := comment.LoadReactions(issue.Repo); err != nil {
+			return nil, err
+		}
+
 		if re, ok := reviews[comment.ReviewID]; ok && re != nil {
 			// If the review is pending only the author can see the comments (except the review is set)
 			if review.ID == 0 {

models/issue_label.go

@@ -47,7 +47,7 @@ type Label struct {
 func GetLabelTemplateFile(name string) ([][3]string, error) {
 	data, err := GetRepoInitFile("label", name)
 	if err != nil {
-		return nil, fmt.Errorf("GetRepoInitFile: %v", err)
+		return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("GetRepoInitFile: %v", err)}
 	}
 
 	lines := strings.Split(string(data), "\n")
@@ -62,7 +62,7 @@ func GetLabelTemplateFile(name string) ([][3]string, error) {
 
 		fields := strings.SplitN(parts[0], " ", 2)
 		if len(fields) != 2 {
-			return nil, fmt.Errorf("line is malformed: %s", line)
+			return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("line is malformed: %s", line)}
 		}
 
 		color := strings.Trim(fields[0], " ")
@@ -70,7 +70,7 @@ func GetLabelTemplateFile(name string) ([][3]string, error) {
 			color = "#" + color
 		}
 		if !LabelColorPattern.MatchString(color) {
-			return nil, fmt.Errorf("bad HTML color code in line: %s", line)
+			return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("bad HTML color code in line: %s", line)}
 		}
 
 		var description string
@@ -167,7 +167,7 @@ func (label *Label) ForegroundColor() template.CSS {
 func loadLabels(labelTemplate string) ([]string, error) {
 	list, err := GetLabelTemplateFile(labelTemplate)
 	if err != nil {
-		return nil, ErrIssueLabelTemplateLoad{labelTemplate, err}
+		return nil, err
 	}
 
 	labels := make([]string, len(list))
@@ -186,7 +186,7 @@ func LoadLabelsFormatted(labelTemplate string) (string, error) {
 func initializeLabels(e Engine, id int64, labelTemplate string, isOrg bool) error {
 	list, err := GetLabelTemplateFile(labelTemplate)
 	if err != nil {
-		return ErrIssueLabelTemplateLoad{labelTemplate, err}
+		return err
 	}
 
 	labels := make([]*Label, len(list))
@@ -764,3 +764,15 @@ func DeleteIssueLabel(issue *Issue, label *Label, doer *User) (err error) {
 
 	return sess.Commit()
 }
+
+func deleteLabelsByRepoID(sess Engine, repoID int64) error {
+	deleteCond := builder.Select("id").From("label").Where(builder.Eq{"label.repo_id": repoID})
+
+	if _, err := sess.In("label_id", deleteCond).
+		Delete(&IssueLabel{}); err != nil {
+		return err
+	}
+
+	_, err := sess.Delete(&Label{RepoID: repoID})
+	return err
+}

models/issue_reaction.go

@@ -178,11 +178,15 @@ func CreateCommentReaction(doer *User, issue *Issue, comment *Comment, content s
 	})
 }
 
-func deleteReaction(e *xorm.Session, opts *ReactionOptions) error {
+func deleteReaction(e Engine, opts *ReactionOptions) error {
 	reaction := &Reaction{
-		Type:    opts.Type,
-		UserID:  opts.Doer.ID,
-		IssueID: opts.Issue.ID,
+		Type: opts.Type,
+	}
+	if opts.Doer != nil {
+		reaction.UserID = opts.Doer.ID
+	}
+	if opts.Issue != nil {
+		reaction.IssueID = opts.Issue.ID
 	}
 	if opts.Comment != nil {
 		reaction.CommentID = opts.Comment.ID

models/list_options.go

@@ -16,13 +16,13 @@ type ListOptions struct {
 	Page     int // start from 1
 }
 
-func (opts ListOptions) getPaginatedSession() *xorm.Session {
+func (opts *ListOptions) getPaginatedSession() *xorm.Session {
 	opts.setDefaultValues()
 
 	return x.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize)
 }
 
-func (opts ListOptions) setSessionPagination(sess *xorm.Session) *xorm.Session {
+func (opts *ListOptions) setSessionPagination(sess *xorm.Session) *xorm.Session {
 	opts.setDefaultValues()
 
 	if opts.PageSize <= 0 {
@@ -31,21 +31,21 @@ func (opts ListOptions) setSessionPagination(sess *xorm.Session) *xorm.Session {
 	return sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize)
 }
 
-func (opts ListOptions) setEnginePagination(e Engine) Engine {
+func (opts *ListOptions) setEnginePagination(e Engine) Engine {
 	opts.setDefaultValues()
 
 	return e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize)
 }
 
 // GetStartEnd returns the start and end of the ListOptions
-func (opts ListOptions) GetStartEnd() (start, end int) {
+func (opts *ListOptions) GetStartEnd() (start, end int) {
 	opts.setDefaultValues()
 	start = (opts.Page - 1) * opts.PageSize
 	end = start + opts.Page
 	return
 }
 
-func (opts ListOptions) setDefaultValues() {
+func (opts *ListOptions) setDefaultValues() {
 	if opts.PageSize <= 0 {
 		opts.PageSize = setting.API.DefaultPagingNum
 	}

models/migrate.go

@@ -39,6 +39,7 @@ func InsertMilestones(ms ...*Milestone) (err error) {
 // InsertIssues insert issues to database
 func InsertIssues(issues ...*Issue) error {
 	sess := x.NewSession()
+	defer sess.Close()
 	if err := sess.Begin(); err != nil {
 		return err
 	}
@@ -194,6 +195,7 @@ func InsertPullRequests(prs ...*PullRequest) error {
 // InsertReleases migrates release
 func InsertReleases(rels ...*Release) error {
 	sess := x.NewSession()
+	defer sess.Close()
 	if err := sess.Begin(); err != nil {
 		return err
 	}

models/migrations/migrations.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 
 	"xorm.io/xorm"
+	"xorm.io/xorm/names"
 )
 
 const minDBVersion = 70 // Gitea 1.5.3
@@ -296,6 +297,8 @@ func EnsureUpToDate(x *xorm.Engine) error {
 
 // Migrate database to current version
 func Migrate(x *xorm.Engine) error {
+	// Set a new clean the default mapper to GonicMapper as that is the default for Gitea.
+	x.SetMapper(names.GonicMapper{})
 	if err := x.Sync(new(Version)); err != nil {
 		return fmt.Errorf("sync: %v", err)
 	}
@@ -334,6 +337,8 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 	// Migrate
 	for i, m := range migrations[v-minDBVersion:] {
 		log.Info("Migration[%d]: %s", v+int64(i), m.Description())
+		// Reset the mapper between each migration - migrations are not supposed to depend on each other
+		x.SetMapper(names.GonicMapper{})
 		if err = m.Migrate(x); err != nil {
 			return fmt.Errorf("do migrate: %v", err)
 		}
@@ -511,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 			return err
 		}
 	case setting.Database.UsePostgreSQL:
+		var originalSequences []string
+		type sequenceData struct {
+			LastValue int  `xorm:"'last_value'"`
+			IsCalled  bool `xorm:"'is_called'"`
+		}
+		sequenceMap := map[string]sequenceData{}
+
+		schema := sess.Engine().Dialect().URI().Schema
+		sess.Engine().SetSchema("")
+		if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, sequence := range originalSequences {
+			sequenceData := sequenceData{}
+			if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
+				log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
+				return err
+			}
+			sequenceMap[sequence] = sequenceData
+
+		}
+
 		// CASCADE causes postgres to drop all the constraints on the old table
 		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
 			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@@ -524,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 		}
 
 		var indices []string
-		schema := sess.Engine().Dialect().URI().Schema
 		sess.Engine().SetSchema("")
 		if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
 			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@@ -540,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 			}
 		}
 
+		var sequences []string
+		sess.Engine().SetSchema("")
+		if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, sequence := range sequences {
+			newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
+			if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+				log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
+				return err
+			}
+			val, ok := sequenceMap[newSequenceName]
+			if newSequenceName == tableName+"_id_seq" {
+				if ok && val.LastValue != 0 {
+					if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+						log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+						return err
+					}
+				} else {
+					// We're going to try to guess this
+					if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+						log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
+						return err
+					}
+				}
+			} else if ok {
+				if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+					log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+					return err
+				}
+			}
+
+		}
+
 	case setting.Database.UseMSSQL:
 		// MSSQL will drop all the constraints on the old table
 		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {

models/migrations/v141.go

@@ -12,7 +12,7 @@ import (
 
 func addKeepActivityPrivateUserColumn(x *xorm.Engine) error {
 	type User struct {
-		KeepActivityPrivate bool
+		KeepActivityPrivate bool `xorm:"NOT NULL DEFAULT false"`
 	}
 
 	if err := x.Sync2(new(User)); err != nil {

models/models.go

@@ -15,12 +15,14 @@ import (
 
 	"code.gitea.io/gitea/modules/setting"
 
-	// Needed for the MySQL driver
-	_ "github.com/go-sql-driver/mysql"
+	"xorm.io/builder"
 	"xorm.io/xorm"
 	"xorm.io/xorm/names"
 	"xorm.io/xorm/schemas"
 
+	// Needed for the MySQL driver
+	_ "github.com/go-sql-driver/mysql"
+
 	// Needed for the Postgresql driver
 	_ "github.com/lib/pq"
 
@@ -145,7 +147,16 @@ func getEngine() (*xorm.Engine, error) {
 		return nil, err
 	}
 
-	engine, err := xorm.NewEngine(setting.Database.Type, connStr)
+	var engine *xorm.Engine
+
+	if setting.Database.UsePostgreSQL && len(setting.Database.Schema) > 0 {
+		// OK whilst we sort out our schema issues - create a schema aware postgres
+		registerPostgresSchemaDriver()
+		engine, err = xorm.NewEngine("postgresschema", connStr)
+	} else {
+		engine, err = xorm.NewEngine(setting.Database.Type, connStr)
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -155,16 +166,6 @@ func getEngine() (*xorm.Engine, error) {
 		engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"})
 	}
 	engine.SetSchema(setting.Database.Schema)
-	if setting.Database.UsePostgreSQL && len(setting.Database.Schema) > 0 {
-		// Add the schema to the search path
-		if _, err := engine.Exec(`SELECT set_config(
-			'search_path',
-			? || ',' || current_setting('search_path'),
-			false)`,
-			setting.Database.Schema); err != nil {
-			return nil, err
-		}
-	}
 	return engine, nil
 }
 
@@ -313,6 +314,13 @@ func DumpDatabase(filePath string, dbType string) error {
 		tbs = append(tbs, t)
 	}
 
+	// temporary fix for v1.13.x (https://github.com/go-gitea/gitea/issues/14069)
+	if _, err := x.Where(builder.IsNull{"keep_activity_private"}).
+		Cols("keep_activity_private").
+		Update(User{KeepActivityPrivate: false}); err != nil {
+		return err
+	}
+
 	type Version struct {
 		ID      int64 `xorm:"pk autoincr"`
 		Version int64

models/oauth2.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 
 	"code.gitea.io/gitea/modules/auth/oauth2"
+	"code.gitea.io/gitea/modules/log"
 )
 
 // OAuth2Provider describes the display values of a single OAuth2 provider
@@ -119,13 +120,28 @@ func InitOAuth2() error {
 	if err := oauth2.Init(x); err != nil {
 		return err
 	}
-	loginSources, _ := GetActiveOAuth2ProviderLoginSources()
+	return initOAuth2LoginSources()
+}
 
+// ResetOAuth2 clears existing OAuth2 providers and loads them from DB
+func ResetOAuth2() error {
+	oauth2.ClearProviders()
+	return initOAuth2LoginSources()
+}
+
+// initOAuth2LoginSources is used to load and register all active OAuth2 providers
+func initOAuth2LoginSources() error {
+	loginSources, _ := GetActiveOAuth2ProviderLoginSources()
 	for _, source := range loginSources {
 		oAuth2Config := source.OAuth2()
 		err := oauth2.RegisterProvider(source.Name, oAuth2Config.Provider, oAuth2Config.ClientID, oAuth2Config.ClientSecret, oAuth2Config.OpenIDConnectAutoDiscoveryURL, oAuth2Config.CustomURLMapping)
 		if err != nil {
-			return err
+			log.Critical("Unable to register source: %s due to Error: %v. This source will be disabled.", source.Name, err)
+			source.IsActived = false
+			if err = UpdateSource(source); err != nil {
+				log.Critical("Unable to update source %s to disable it. Error: %v", err)
+				return err
+			}
 		}
 	}
 	return nil

models/oauth2_application.go

@@ -233,7 +233,7 @@ func deleteOAuth2Application(sess *xorm.Session, id, userid int64) error {
 	if deleted, err := sess.Delete(&OAuth2Application{ID: id, UID: userid}); err != nil {
 		return err
 	} else if deleted == 0 {
-		return fmt.Errorf("cannot find oauth2 application")
+		return ErrOAuthApplicationNotFound{ID: id}
 	}
 	codes := make([]*OAuth2AuthorizationCode, 0)
 	// delete correlating auth codes
@@ -259,6 +259,7 @@ func deleteOAuth2Application(sess *xorm.Session, id, userid int64) error {
 // DeleteOAuth2Application deletes the application with the given id and the grants and auth codes related to it. It checks if the userid was the creator of the app.
 func DeleteOAuth2Application(id, userid int64) error {
 	sess := x.NewSession()
+	defer sess.Close()
 	if err := sess.Begin(); err != nil {
 		return err
 	}

models/release.go

@@ -54,7 +54,11 @@ func (r *Release) loadAttributes(e Engine) error {
 	if r.Publisher == nil {
 		r.Publisher, err = getUserByID(e, r.PublisherID)
 		if err != nil {
-			return err
+			if IsErrUserNotExist(err) {
+				r.Publisher = NewGhostUser()
+			} else {
+				return err
+			}
 		}
 	}
 	return getReleaseAttachments(e, r)

models/repo.go

@@ -426,6 +426,7 @@ func (repo *Repository) innerAPIFormat(e Engine, mode AccessMode, isParent bool)
 		HTMLURL:                   repo.HTMLURL(),
 		SSHURL:                    cloneLink.SSH,
 		CloneURL:                  cloneLink.HTTPS,
+		OriginalURL:               repo.SanitizedOriginalURL(),
 		Website:                   repo.Website,
 		Stars:                     repo.NumStars,
 		Forks:                     repo.NumForks,
@@ -1289,11 +1290,44 @@ func IncrementRepoForkNum(ctx DBContext, repoID int64) error {
 }
 
 // TransferOwnership transfers all corresponding setting from old user to new one.
-func TransferOwnership(doer *User, newOwnerName string, repo *Repository) error {
+func TransferOwnership(doer *User, newOwnerName string, repo *Repository) (err error) {
+	repoRenamed := false
+	wikiRenamed := false
+	oldOwnerName := doer.Name
+
+	defer func() {
+		if !repoRenamed && !wikiRenamed {
+			return
+		}
+
+		recoverErr := recover()
+		if err == nil && recoverErr == nil {
+			return
+		}
+
+		if repoRenamed {
+			if err := os.Rename(RepoPath(newOwnerName, repo.Name), RepoPath(oldOwnerName, repo.Name)); err != nil {
+				log.Critical("Unable to move repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, RepoPath(newOwnerName, repo.Name), RepoPath(oldOwnerName, repo.Name), err)
+			}
+		}
+
+		if wikiRenamed {
+			if err := os.Rename(WikiPath(newOwnerName, repo.Name), WikiPath(oldOwnerName, repo.Name)); err != nil {
+				log.Critical("Unable to move wiki for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, WikiPath(newOwnerName, repo.Name), WikiPath(oldOwnerName, repo.Name), err)
+			}
+		}
+
+		if recoverErr != nil {
+			log.Error("Panic within TransferOwnership: %v\n%s", recoverErr, log.Stack(2))
+			panic(recoverErr)
+		}
+	}()
+
 	newOwner, err := GetUserByName(newOwnerName)
 	if err != nil {
 		return fmt.Errorf("get new owner '%s': %v", newOwnerName, err)
 	}
+	newOwnerName = newOwner.Name // ensure capitalisation matches
 
 	// Check if new owner has repository with same name.
 	has, err := IsRepositoryExist(newOwner, repo.Name)
@@ -1310,6 +1344,7 @@ func TransferOwnership(doer *User, newOwnerName string, repo *Repository) error
 	}
 
 	oldOwner := repo.Owner
+	oldOwnerName = oldOwner.Name
 
 	// Note: we have to set value here to make sure recalculate accesses is based on
 	// new owner.
@@ -1369,9 +1404,9 @@ func TransferOwnership(doer *User, newOwnerName string, repo *Repository) error
 	}
 
 	// Update repository count.
-	if _, err = sess.Exec("UPDATE `user` SET num_repos=num_repos+1 WHERE id=?", newOwner.ID); err != nil {
+	if _, err := sess.Exec("UPDATE `user` SET num_repos=num_repos+1 WHERE id=?", newOwner.ID); err != nil {
 		return fmt.Errorf("increase new owner repository count: %v", err)
-	} else if _, err = sess.Exec("UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", oldOwner.ID); err != nil {
+	} else if _, err := sess.Exec("UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", oldOwner.ID); err != nil {
 		return fmt.Errorf("decrease old owner repository count: %v", err)
 	}
 
@@ -1381,7 +1416,7 @@ func TransferOwnership(doer *User, newOwnerName string, repo *Repository) error
 
 	// Remove watch for organization.
 	if oldOwner.IsOrganization() {
-		if err = watchRepo(sess, oldOwner.ID, repo.ID, false); err != nil {
+		if err := watchRepo(sess, oldOwner.ID, repo.ID, false); err != nil {
 			return fmt.Errorf("watchRepo [false]: %v", err)
 		}
 	}
@@ -1393,16 +1428,18 @@ func TransferOwnership(doer *User, newOwnerName string, repo *Repository) error
 		return fmt.Errorf("Failed to create dir %s: %v", dir, err)
 	}
 
-	if err = os.Rename(RepoPath(oldOwner.Name, repo.Name), RepoPath(newOwner.Name, repo.Name)); err != nil {
+	if err := os.Rename(RepoPath(oldOwner.Name, repo.Name), RepoPath(newOwner.Name, repo.Name)); err != nil {
 		return fmt.Errorf("rename repository directory: %v", err)
 	}
+	repoRenamed = true
 
 	// Rename remote wiki repository to new path and delete local copy.
 	wikiPath := WikiPath(oldOwner.Name, repo.Name)
 	if com.IsExist(wikiPath) {
-		if err = os.Rename(wikiPath, WikiPath(newOwner.Name, repo.Name)); err != nil {
+		if err := os.Rename(wikiPath, WikiPath(newOwner.Name, repo.Name)); err != nil {
 			return fmt.Errorf("rename repository wiki: %v", err)
 		}
+		wikiRenamed = true
 	}
 
 	// If there was previously a redirect at this location, remove it.
@@ -1599,26 +1636,27 @@ func UpdateRepositoryUnits(repo *Repository, units []RepoUnit, deleteUnitTypes [
 }
 
 // DeleteRepository deletes a repository for a user or organization.
+// make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock)
 func DeleteRepository(doer *User, uid, repoID int64) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
 	// In case is a organization.
-	org, err := GetUserByID(uid)
+	org, err := getUserByID(sess, uid)
 	if err != nil {
 		return err
 	}
 	if org.IsOrganization() {
-		if err = org.GetTeams(&SearchTeamOptions{}); err != nil {
+		if err = org.getTeams(sess); err != nil {
 			return err
 		}
 	}
 
-	sess := x.NewSession()
-	defer sess.Close()
-	if err = sess.Begin(); err != nil {
-		return err
-	}
-
-	repo := &Repository{ID: repoID, OwnerID: uid}
-	has, err := sess.Get(repo)
+	repo := &Repository{OwnerID: uid}
+	has, err := sess.ID(repoID).Get(repo)
 	if err != nil {
 		return err
 	} else if !has {
@@ -1691,6 +1729,10 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 		return fmt.Errorf("deleteBeans: %v", err)
 	}
 
+	if err := deleteLabelsByRepoID(sess, repoID); err != nil {
+		return err
+	}
+
 	// Delete Issues and related objects
 	var attachmentPaths []string
 	if attachmentPaths, err = deleteIssuesByRepoID(sess, repoID); err != nil {
@@ -1767,14 +1809,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 	}
 
 	if err = sess.Commit(); err != nil {
-		sess.Close()
-		if len(deployKeys) > 0 {
-			// We need to rewrite the public keys because the commit failed
-			if err2 := RewriteAllPublicKeys(); err2 != nil {
-				return fmt.Errorf("Commit: %v SSH Keys: %v", err, err2)
-			}
-		}
-		return fmt.Errorf("Commit: %v", err)
+		return err
 	}
 
 	sess.Close()

models/repo_permission.go

@@ -271,6 +271,27 @@ func getUserRepoPermission(e Engine, repo *Repository, user *User) (perm Permiss
 	return
 }
 
+// IsUserRealRepoAdmin check if this user is real repo admin
+func IsUserRealRepoAdmin(repo *Repository, user *User) (bool, error) {
+	if repo.OwnerID == user.ID {
+		return true, nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := repo.getOwner(sess); err != nil {
+		return false, err
+	}
+
+	accessMode, err := accessLevel(sess, user, repo)
+	if err != nil {
+		return false, err
+	}
+
+	return accessMode >= AccessModeAdmin, nil
+}
+
 // IsUserRepoAdmin return true if user has admin right of a repo
 func IsUserRepoAdmin(repo *Repository, user *User) (bool, error) {
 	return isUserRepoAdmin(x, repo, user)

models/sql_postgres_with_schema.go

@@ -0,0 +1,75 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"sync"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/lib/pq"
+	"xorm.io/xorm/dialects"
+)
+
+var registerOnce sync.Once
+
+func registerPostgresSchemaDriver() {
+	registerOnce.Do(func() {
+		sql.Register("postgresschema", &postgresSchemaDriver{})
+		dialects.RegisterDriver("postgresschema", dialects.QueryDriver("postgres"))
+	})
+}
+
+type postgresSchemaDriver struct {
+	pq.Driver
+}
+
+// Open opens a new connection to the database. name is a connection string.
+// This function opens the postgres connection in the default manner but immediately
+// runs set_config to set the search_path appropriately
+func (d *postgresSchemaDriver) Open(name string) (driver.Conn, error) {
+	conn, err := d.Driver.Open(name)
+	if err != nil {
+		return conn, err
+	}
+	schemaValue, _ := driver.String.ConvertValue(setting.Database.Schema)
+
+	// golangci lint is incorrect here - there is no benefit to using driver.ExecerContext here
+	// and in any case pq does not implement it
+	if execer, ok := conn.(driver.Execer); ok { //nolint
+		_, err := execer.Exec(`SELECT set_config(
+			'search_path',
+			$1 || ',' || current_setting('search_path'),
+			false)`, []driver.Value{schemaValue}) //nolint
+		if err != nil {
+			_ = conn.Close()
+			return nil, err
+		}
+		return conn, nil
+	}
+
+	stmt, err := conn.Prepare(`SELECT set_config(
+		'search_path',
+		$1 || ',' || current_setting('search_path'),
+		false)`)
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	defer stmt.Close()
+
+	// driver.String.ConvertValue will never return err for string
+
+	// golangci lint is incorrect here - there is no benefit to using stmt.ExecWithContext here
+	_, err = stmt.Exec([]driver.Value{schemaValue}) //nolint
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+
+	return conn, nil
+}

models/task.go

@@ -147,6 +147,27 @@ func GetMigratingTask(repoID int64) (*Task, error) {
 	return &task, nil
 }
 
+// GetMigratingTaskByID returns the migrating task by repo's id
+func GetMigratingTaskByID(id, doerID int64) (*Task, *migration.MigrateOptions, error) {
+	var task = Task{
+		ID:     id,
+		DoerID: doerID,
+		Type:   structs.TaskTypeMigrateRepo,
+	}
+	has, err := x.Get(&task)
+	if err != nil {
+		return nil, nil, err
+	} else if !has {
+		return nil, nil, ErrTaskDoesNotExist{id, 0, task.Type}
+	}
+
+	var opts migration.MigrateOptions
+	if err := json.Unmarshal([]byte(task.PayloadContent), &opts); err != nil {
+		return nil, nil, err
+	}
+	return &task, &opts, nil
+}
+
 // FindTaskOptions find all tasks
 type FindTaskOptions struct {
 	Status int

models/topic.go

@@ -197,10 +197,13 @@ func FindTopics(opts *FindTopicOptions) (topics []*Topic, err error) {
 
 // GetRepoTopicByName retrives topic from name for a repo if it exist
 func GetRepoTopicByName(repoID int64, topicName string) (*Topic, error) {
+	return getRepoTopicByName(x, repoID, topicName)
+}
+func getRepoTopicByName(e Engine, repoID int64, topicName string) (*Topic, error) {
 	var cond = builder.NewCond()
 	var topic Topic
 	cond = cond.And(builder.Eq{"repo_topic.repo_id": repoID}).And(builder.Eq{"topic.name": topicName})
-	sess := x.Table("topic").Where(cond)
+	sess := e.Table("topic").Where(cond)
 	sess.Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id")
 	has, err := sess.Get(&topic)
 	if has {
@@ -211,7 +214,13 @@ func GetRepoTopicByName(repoID int64, topicName string) (*Topic, error) {
 
 // AddTopic adds a topic name to a repository (if it does not already have it)
 func AddTopic(repoID int64, topicName string) (*Topic, error) {
-	topic, err := GetRepoTopicByName(repoID, topicName)
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return nil, err
+	}
+
+	topic, err := getRepoTopicByName(sess, repoID, topicName)
 	if err != nil {
 		return nil, err
 	}
@@ -220,7 +229,25 @@ func AddTopic(repoID int64, topicName string) (*Topic, error) {
 		return topic, nil
 	}
 
-	return addTopicByNameToRepo(x, repoID, topicName)
+	topic, err = addTopicByNameToRepo(sess, repoID, topicName)
+	if err != nil {
+		return nil, err
+	}
+
+	topicNames := make([]string, 0, 25)
+	if err := sess.Select("name").Table("topic").
+		Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id").
+		Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil {
+		return nil, err
+	}
+
+	if _, err := sess.ID(repoID).Cols("topics").Update(&Repository{
+		Topics: topicNames,
+	}); err != nil {
+		return nil, err
+	}
+
+	return topic, sess.Commit()
 }
 
 // DeleteTopic removes a topic name from a repository (if it has it)

models/user.go

@@ -40,7 +40,6 @@ import (
 	"golang.org/x/crypto/scrypt"
 	"golang.org/x/crypto/ssh"
 	"xorm.io/builder"
-	"xorm.io/xorm"
 )
 
 // UserType defines the user type
@@ -191,9 +190,6 @@ func (u *User) BeforeUpdate() {
 		if len(u.AvatarEmail) == 0 {
 			u.AvatarEmail = u.Email
 		}
-		if len(u.AvatarEmail) > 0 && u.Avatar == "" {
-			u.Avatar = base.HashEmail(u.AvatarEmail)
-		}
 	}
 
 	u.LowerName = strings.ToLower(u.Name)
@@ -554,6 +550,7 @@ func (u *User) GetOwnedOrganizations() (err error) {
 }
 
 // GetOrganizations returns paginated organizations that user belongs to.
+// TODO: does not respect All and show orgs you privately participate
 func (u *User) GetOrganizations(opts *SearchOrganizationsOptions) error {
 	sess := x.NewSession()
 	defer sess.Close()
@@ -730,6 +727,7 @@ var (
 		"assets",
 		"attachments",
 		"avatars",
+		"captcha",
 		"commits",
 		"debug",
 		"error",
@@ -824,6 +822,10 @@ func CreateUser(u *User) (err error) {
 		return ErrEmailAlreadyUsed{u.Email}
 	}
 
+	if err = ValidateEmail(u.Email); err != nil {
+		return err
+	}
+
 	isExist, err = isEmailUsed(sess, u.Email)
 	if err != nil {
 		return err
@@ -835,7 +837,6 @@ func CreateUser(u *User) (err error) {
 
 	u.LowerName = strings.ToLower(u.Name)
 	u.AvatarEmail = u.Email
-	u.Avatar = base.HashEmail(u.AvatarEmail)
 	if u.Rands, err = GetUserSalt(); err != nil {
 		return err
 	}
@@ -922,6 +923,7 @@ func VerifyActiveEmailCode(code, email string) *EmailAddress {
 
 // ChangeUserName changes all corresponding setting from old user name to new one.
 func ChangeUserName(u *User, newUserName string) (err error) {
+	oldUserName := u.Name
 	if err = IsUsableUsername(newUserName); err != nil {
 		return err
 	}
@@ -939,16 +941,24 @@ func ChangeUserName(u *User, newUserName string) (err error) {
 		return err
 	}
 
-	if _, err = sess.Exec("UPDATE `repository` SET owner_name=? WHERE owner_name=?", newUserName, u.Name); err != nil {
+	if _, err = sess.Exec("UPDATE `repository` SET owner_name=? WHERE owner_name=?", newUserName, oldUserName); err != nil {
 		return fmt.Errorf("Change repo owner name: %v", err)
 	}
 
 	// Do not fail if directory does not exist
-	if err = os.Rename(UserPath(u.Name), UserPath(newUserName)); err != nil && !os.IsNotExist(err) {
+	if err = os.Rename(UserPath(oldUserName), UserPath(newUserName)); err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("Rename user directory: %v", err)
 	}
 
-	return sess.Commit()
+	if err = sess.Commit(); err != nil {
+		if err2 := os.Rename(UserPath(newUserName), UserPath(oldUserName)); err2 != nil && !os.IsNotExist(err2) {
+			log.Critical("Unable to rollback directory change during failed username change from: %s to: %s. DB Error: %v. Filesystem Error: %v", oldUserName, newUserName, err, err2)
+			return fmt.Errorf("failed to rollback directory change during failed username change from: %s to: %s. DB Error: %w. Filesystem Error: %v", oldUserName, newUserName, err, err2)
+		}
+		return err
+	}
+
+	return nil
 }
 
 // checkDupEmail checks whether there are the same email with the user
@@ -967,8 +977,12 @@ func checkDupEmail(e Engine, u *User) error {
 	return nil
 }
 
-func updateUser(e Engine, u *User) error {
-	_, err := e.ID(u.ID).AllCols().Update(u)
+func updateUser(e Engine, u *User) (err error) {
+	u.Email = strings.ToLower(u.Email)
+	if err = ValidateEmail(u.Email); err != nil {
+		return err
+	}
+	_, err = e.ID(u.ID).AllCols().Update(u)
 	return err
 }
 
@@ -988,13 +1002,21 @@ func updateUserCols(e Engine, u *User, cols ...string) error {
 }
 
 // UpdateUserSetting updates user's settings.
-func UpdateUserSetting(u *User) error {
+func UpdateUserSetting(u *User) (err error) {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err = sess.Begin(); err != nil {
+		return err
+	}
 	if !u.IsOrganization() {
-		if err := checkDupEmail(x, u); err != nil {
+		if err = checkDupEmail(sess, u); err != nil {
 			return err
 		}
 	}
-	return updateUser(x, u)
+	if err = updateUser(sess, u); err != nil {
+		return err
+	}
+	return sess.Commit()
 }
 
 // deleteBeans deletes all given beans, beans should contain delete conditions.
@@ -1007,8 +1029,7 @@ func deleteBeans(e Engine, beans ...interface{}) (err error) {
 	return nil
 }
 
-// FIXME: need some kind of mechanism to record failure. HINT: system notice
-func deleteUser(e *xorm.Session, u *User) error {
+func deleteUser(e Engine, u *User) error {
 	// Note: A user owns any repository or belongs to any organization
 	//	cannot perform delete operation.
 
@@ -1102,6 +1123,16 @@ func deleteUser(e *xorm.Session, u *User) error {
 	// ***** END: PublicKey *****
 
 	// ***** START: GPGPublicKey *****
+	keys, err := listGPGKeys(e, u.ID, ListOptions{})
+	if err != nil {
+		return fmt.Errorf("ListGPGKeys: %v", err)
+	}
+	// Delete GPGKeyImport(s).
+	for _, key := range keys {
+		if _, err = e.Delete(&GPGKeyImport{KeyID: key.KeyID}); err != nil {
+			return fmt.Errorf("deleteGPGKeyImports: %v", err)
+		}
+	}
 	if _, err = e.Delete(&GPGKey{OwnerID: u.ID}); err != nil {
 		return fmt.Errorf("deleteGPGKeys: %v", err)
 	}
@@ -1122,18 +1153,21 @@ func deleteUser(e *xorm.Session, u *User) error {
 		return fmt.Errorf("Delete: %v", err)
 	}
 
-	// FIXME: system notice
 	// Note: There are something just cannot be roll back,
 	//	so just keep error logs of those operations.
 	path := UserPath(u.Name)
-	if err := util.RemoveAll(path); err != nil {
-		return fmt.Errorf("Failed to RemoveAll %s: %v", path, err)
+	if err = util.RemoveAll(path); err != nil {
+		err = fmt.Errorf("Failed to RemoveAll %s: %v", path, err)
+		_ = createNotice(e, NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err))
+		return err
 	}
 
 	if len(u.Avatar) > 0 {
 		avatarPath := u.CustomAvatarRelativePath()
-		if err := storage.Avatars.Delete(avatarPath); err != nil {
-			return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
+		if err = storage.Avatars.Delete(avatarPath); err != nil {
+			err = fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
+			_ = createNotice(e, NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err))
+			return err
 		}
 	}
 
@@ -1589,20 +1623,34 @@ func deleteKeysMarkedForDeletion(keys []string) (bool, error) {
 func addLdapSSHPublicKeys(usr *User, s *LoginSource, sshPublicKeys []string) bool {
 	var sshKeysNeedUpdate bool
 	for _, sshKey := range sshPublicKeys {
-		_, _, _, _, err := ssh.ParseAuthorizedKey([]byte(sshKey))
-		if err == nil {
-			sshKeyName := fmt.Sprintf("%s-%s", s.Name, sshKey[0:40])
-			if _, err := AddPublicKey(usr.ID, sshKeyName, sshKey, s.ID); err != nil {
+		var err error
+		found := false
+		keys := []byte(sshKey)
+	loop:
+		for len(keys) > 0 && err == nil {
+			var out ssh.PublicKey
+			// We ignore options as they are not relevant to Gitea
+			out, _, _, keys, err = ssh.ParseAuthorizedKey(keys)
+			if err != nil {
+				break loop
+			}
+			found = true
+			marshalled := string(ssh.MarshalAuthorizedKey(out))
+			marshalled = marshalled[:len(marshalled)-1]
+			sshKeyName := fmt.Sprintf("%s-%s", s.Name, ssh.FingerprintSHA256(out))
+
+			if _, err := AddPublicKey(usr.ID, sshKeyName, marshalled, s.ID); err != nil {
 				if IsErrKeyAlreadyExist(err) {
-					log.Trace("addLdapSSHPublicKeys[%s]: LDAP Public SSH Key %s already exists for user", s.Name, usr.Name)
+					log.Trace("addLdapSSHPublicKeys[%s]: LDAP Public SSH Key %s already exists for user", sshKeyName, usr.Name)
 				} else {
-					log.Error("addLdapSSHPublicKeys[%s]: Error adding LDAP Public SSH Key for user %s: %v", s.Name, usr.Name, err)
+					log.Error("addLdapSSHPublicKeys[%s]: Error adding LDAP Public SSH Key for user %s: %v", sshKeyName, usr.Name, err)
 				}
 			} else {
-				log.Trace("addLdapSSHPublicKeys[%s]: Added LDAP Public SSH Key for user %s", s.Name, usr.Name)
+				log.Trace("addLdapSSHPublicKeys[%s]: Added LDAP Public SSH Key for user %s", sshKeyName, usr.Name)
 				sshKeysNeedUpdate = true
 			}
-		} else {
+		}
+		if !found && err != nil {
 			log.Warn("addLdapSSHPublicKeys[%s]: Skipping invalid LDAP Public SSH Key for user %s: %v", s.Name, usr.Name, sshKey)
 		}
 	}

models/user_avatar.go

@@ -39,12 +39,10 @@ func (u *User) generateRandomAvatar(e Engine) error {
 	if err != nil {
 		return fmt.Errorf("RandomImage: %v", err)
 	}
-	// NOTICE for random avatar, it still uses id as avatar name, but custom avatar use md5
-	// since random image is not a user's photo, there is no security for enumable
-	if u.Avatar == "" {
-		u.Avatar = fmt.Sprintf("%d", u.ID)
-	}
 
+	u.Avatar = base.HashEmail(seed)
+
+	// Don't share the images so that we can delete them easily
 	if err := storage.SaveFrom(storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error {
 		if err := png.Encode(w, img); err != nil {
 			log.Error("Encode: %v", err)
@@ -134,7 +132,7 @@ func (u *User) UploadAvatar(data []byte) error {
 	// Otherwise, if any of the users delete his avatar
 	// Other users will lose their avatars too.
 	u.Avatar = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data)))))
-	if err = updateUser(sess, u); err != nil {
+	if err = updateUserCols(sess, u, "use_custom_avatar", "avatar"); err != nil {
 		return fmt.Errorf("updateUser: %v", err)
 	}
 

models/user_heatmap_test.go

@@ -17,7 +17,7 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
 		CountResult int
 		JSONResult  string
 	}{
-		{2, 1, `[{"timestamp":1571616000,"contributions":1}]`},
+		{2, 1, `[{"timestamp":1603152000,"contributions":1}]`},
 		{3, 0, `[]`},
 	}
 	// Prepare

models/user_mail.go

@@ -8,6 +8,7 @@ package models
 import (
 	"errors"
 	"fmt"
+	"net/mail"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
@@ -32,6 +33,19 @@ type EmailAddress struct {
 	IsPrimary   bool `xorm:"-"`
 }
 
+// ValidateEmail check if email is a allowed address
+func ValidateEmail(email string) error {
+	if len(email) == 0 {
+		return nil
+	}
+
+	if _, err := mail.ParseAddress(email); err != nil {
+		return ErrEmailInvalid{email}
+	}
+
+	return nil
+}
+
 // GetEmailAddresses returns all email addresses belongs to given user.
 func GetEmailAddresses(uid int64) ([]*EmailAddress, error) {
 	emails := make([]*EmailAddress, 0, 5)
@@ -143,6 +157,10 @@ func addEmailAddress(e Engine, email *EmailAddress) error {
 		return ErrEmailAlreadyUsed{email.Email}
 	}
 
+	if err = ValidateEmail(email.Email); err != nil {
+		return err
+	}
+
 	_, err = e.Insert(email)
 	return err
 }
@@ -167,6 +185,9 @@ func AddEmailAddresses(emails []*EmailAddress) error {
 		} else if used {
 			return ErrEmailAlreadyUsed{emails[i].Email}
 		}
+		if err = ValidateEmail(emails[i].Email); err != nil {
+			return err
+		}
 	}
 
 	if _, err := x.Insert(emails); err != nil {

models/user_test.go

@@ -346,6 +346,21 @@ func TestCreateUser(t *testing.T) {
 	assert.NoError(t, DeleteUser(user))
 }
 
+func TestCreateUserInvalidEmail(t *testing.T) {
+	user := &User{
+		Name:               "GiteaBot",
+		Email:              "GiteaBot@gitea.io\r\n",
+		Passwd:             ";p['////..-++']",
+		IsAdmin:            false,
+		Theme:              setting.UI.DefaultTheme,
+		MustChangePassword: false,
+	}
+
+	err := CreateUser(user)
+	assert.Error(t, err)
+	assert.True(t, IsErrEmailInvalid(err))
+}
+
 func TestCreateUser_Issue5882(t *testing.T) {
 
 	// Init settings
@@ -406,3 +421,71 @@ func TestGetMaileableUsersByIDs(t *testing.T) {
 		assert.Equal(t, results[1].ID, 4)
 	}
 }
+
+func TestAddLdapSSHPublicKeys(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	user := AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)
+	s := &LoginSource{ID: 1}
+
+	testCases := []struct {
+		keyString   string
+		number      int
+		keyContents []string
+	}{
+		{
+			keyString: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM= nocomment\n",
+			number:    1,
+			keyContents: []string{
+				"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM=",
+			},
+		},
+		{
+			keyString: `ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM= nocomment
+ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag= nocomment`,
+			number: 2,
+			keyContents: []string{
+				"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM=",
+				"ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag=",
+			},
+		},
+		{
+			keyString: `ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM= nocomment
+# comment asmdna,ndp
+ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag= nocomment`,
+			number: 2,
+			keyContents: []string{
+				"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM=",
+				"ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag=",
+			},
+		},
+		{
+			keyString: `ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM= nocomment
+382488320jasdj1lasmva/vasodifipi4193-fksma.cm
+ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag= nocomment`,
+			number: 2,
+			keyContents: []string{
+				"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4cn+iXnA4KvcQYSV88vGn0Yi91vG47t1P7okprVmhNTkipNRIHWr6WdCO4VDr/cvsRkuVJAsLO2enwjGWWueOO6BodiBgyAOZ/5t5nJNMCNuLGT5UIo/RI1b0WRQwxEZTRjt6mFNw6lH14wRd8ulsr9toSWBPMOGWoYs1PDeDL0JuTjL+tr1SZi/EyxCngpYszKdXllJEHyI79KQgeD0Vt3pTrkbNVTOEcCNqZePSVmUH8X8Vhugz3bnE0/iE9Pb5fkWO9c4AnM1FgI/8Bvp27Fw2ShryIXuR6kKvUqhVMTuOSDHwu6A8jLE5Owt3GAYugDpDYuwTVNGrHLXKpPzrGGPE/jPmaLCMZcsdkec95dYeU3zKODEm8UQZFhmJmDeWVJ36nGrGZHL4J5aTTaeFUJmmXDaJYiJ+K2/ioKgXqnXvltu0A9R8/LGy4nrTJRr4JMLuJFoUXvGm1gXQ70w2LSpk6yl71RNC0hCtsBe8BP8IhYCM0EP5jh7eCMQZNvM=",
+				"ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ibZ2OkQ3S0SqDIa0HXSEJ1zaExQdmbO+Ux/wsytWZmCczWOVsaszBZSl90q8UnWlSH6P+/YA+RWJm5SFtuV9PtGIhyZgoNuz5kBQ7K139wuQsecdKktISwTakzAAAAFQCzKsO2JhNKlL+wwwLGOcLffoAmkwAAAIBpK7/3xvduajLBD/9vASqBQIHrgK2J+wiQnIb/Wzy0UsVmvfn8A+udRbBo+csM8xrSnlnlJnjkJS3qiM5g+eTwsLIV1IdKPEwmwB+VcP53Cw6lSyWyJcvhFb0N6s08NZysLzvj0N+ZC/FnhKTLzIyMtkHf/IrPCwlM+pV/M/96YgAAAIEAqQcGn9CKgzgPaguIZooTAOQdvBLMI5y0bQjOW6734XOpqQGf/Kra90wpoasLKZjSYKNPjE+FRUOrStLrxcNs4BeVKhy2PYTRnybfYVk1/dmKgH6P1YSRONsGKvTsH6c5IyCRG0ncCgYeF8tXppyd642982daopE7zQ/NPAnJfag=",
+			},
+		},
+	}
+
+	for i, kase := range testCases {
+		s.ID = int64(i) + 20
+		addLdapSSHPublicKeys(user, s, []string{kase.keyString})
+		keys, err := ListPublicLdapSSHKeys(user.ID, s.ID)
+		assert.NoError(t, err)
+		if err != nil {
+			continue
+		}
+		assert.Equal(t, kase.number, len(keys))
+
+		for _, key := range keys {
+			assert.Contains(t, kase.keyContents, key.Content)
+		}
+		for _, key := range keys {
+			DeletePublicKey(user, key.ID)
+		}
+	}
+}

modules/analyze/vendor.go

@@ -0,0 +1,70 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package analyze
+
+import (
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/go-enry/go-enry/v2/data"
+)
+
+var isVendorRegExp *regexp.Regexp
+
+func init() {
+	matchers := data.VendorMatchers
+
+	caretStrings := make([]string, 0, 10)
+	caretShareStrings := make([]string, 0, 10)
+
+	matcherStrings := make([]string, 0, len(matchers))
+	for _, matcher := range matchers {
+		str := matcher.String()
+		if str[0] == '^' {
+			caretStrings = append(caretStrings, str[1:])
+		} else if str[0:5] == "(^|/)" {
+			caretShareStrings = append(caretShareStrings, str[5:])
+		} else {
+			matcherStrings = append(matcherStrings, str)
+		}
+	}
+
+	sort.Strings(caretShareStrings)
+	sort.Strings(caretStrings)
+	sort.Strings(matcherStrings)
+
+	sb := &strings.Builder{}
+	sb.WriteString("(?:^(?:")
+	sb.WriteString(caretStrings[0])
+	for _, matcher := range caretStrings[1:] {
+		sb.WriteString(")|(?:")
+		sb.WriteString(matcher)
+	}
+	sb.WriteString("))")
+	sb.WriteString("|")
+	sb.WriteString("(?:(?:^|/)(?:")
+	sb.WriteString(caretShareStrings[0])
+	for _, matcher := range caretShareStrings[1:] {
+		sb.WriteString(")|(?:")
+		sb.WriteString(matcher)
+	}
+	sb.WriteString("))")
+	sb.WriteString("|")
+	sb.WriteString("(?:")
+	sb.WriteString(matcherStrings[0])
+	for _, matcher := range matcherStrings[1:] {
+		sb.WriteString(")|(?:")
+		sb.WriteString(matcher)
+	}
+	sb.WriteString(")")
+	combined := sb.String()
+	isVendorRegExp = regexp.MustCompile(combined)
+}
+
+// IsVendor returns whether or not path is a vendor path.
+func IsVendor(path string) bool {
+	return isVendorRegExp.MatchString(path)
+}

modules/analyze/vendor_test.go

@@ -0,0 +1,42 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package analyze
+
+import "testing"
+
+func TestIsVendor(t *testing.T) {
+	tests := []struct {
+		path string
+		want bool
+	}{
+		{"cache/", true},
+		{"random/cache/", true},
+		{"cache", false},
+		{"dependencies/", true},
+		{"Dependencies/", true},
+		{"dependency/", false},
+		{"dist/", true},
+		{"dist", false},
+		{"random/dist/", true},
+		{"random/dist", false},
+		{"deps/", true},
+		{"configure", true},
+		{"a/configure", true},
+		{"config.guess", true},
+		{"config.guess/", false},
+		{".vscode/", true},
+		{"doc/_build/", true},
+		{"a/docs/_build/", true},
+		{"a/dasdocs/_build-vsdoc.js", true},
+		{"a/dasdocs/_build-vsdoc.j", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.path, func(t *testing.T) {
+			if got := IsVendor(tt.path); got != tt.want {
+				t.Errorf("IsVendor() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

modules/auth/oauth2/oauth2.go

@@ -118,6 +118,11 @@ func RemoveProvider(providerName string) {
 	delete(goth.GetProviders(), providerName)
 }
 
+// ClearProviders clears all OAuth2 providers from the goth lib
+func ClearProviders() {
+	goth.ClearProviders()
+}
+
 // used to create different types of goth providers
 func createProvider(providerName, providerType, clientID, clientSecret, openIDConnectAutoDiscoveryURL string, customURLMapping *CustomURLMapping) (goth.Provider, error) {
 	callbackURL := setting.AppURL + "user/oauth2/" + url.PathEscape(providerName) + "/callback"

modules/auth/pam/pam.go

@@ -12,6 +12,9 @@ import (
 	"github.com/msteinert/pam"
 )
 
+// Supported is true when built with PAM
+var Supported = true
+
 // Auth pam auth service
 func Auth(serviceName, userName, passwd string) (string, error) {
 	t, err := pam.StartFunc(serviceName, userName, func(s pam.Style, msg string) (string, error) {

modules/auth/pam/pam_stub.go

@@ -10,6 +10,9 @@ import (
 	"errors"
 )
 
+// Supported is false when built without PAM
+var Supported = false
+
 // Auth not supported lack of pam tag
 func Auth(serviceName, userName, passwd string) (string, error) {
 	return "", errors.New("PAM not supported")

modules/auth/repo_form.go

@@ -102,6 +102,9 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string, user *models
 			u.User = url.UserPassword(authUsername, authPassword)
 		}
 		remoteAddr = u.String()
+		if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteAddr, "%0d") || strings.Contains(remoteAddr, "%0a")) {
+			return "", models.ErrInvalidCloneAddr{IsURLError: true}
+		}
 	} else if !user.CanImportLocal() {
 		return "", models.ErrInvalidCloneAddr{IsPermissionDenied: true}
 	} else if !com.IsDir(remoteAddr) {

modules/auth/user_form.go

@@ -199,7 +199,6 @@ func (f *AccessTokenForm) Validate(ctx *macaron.Context, errs binding.Errors) bi
 type UpdateProfileForm struct {
 	Name                string `binding:"AlphaDashDot;MaxSize(40)"`
 	FullName            string `binding:"MaxSize(100)"`
-	Email               string `binding:"Required;Email;MaxSize(254)"`
 	KeepEmailPrivate    bool
 	Website             string `binding:"ValidUrl;MaxSize(255)"`
 	Location            string `binding:"MaxSize(50)"`

modules/base/tool.go

@@ -10,6 +10,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -65,6 +66,11 @@ func BasicAuthDecode(encoded string) (string, string, error) {
 	}
 
 	auth := strings.SplitN(string(s), ":", 2)
+
+	if len(auth) != 2 {
+		return "", "", errors.New("invalid basic authentication")
+	}
+
 	return auth[0], auth[1], nil
 }
 

modules/base/tool_test.go

@@ -46,6 +46,12 @@ func TestBasicAuthDecode(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "foo", user)
 	assert.Equal(t, "bar", pass)
+
+	_, _, err = BasicAuthDecode("aW52YWxpZA==")
+	assert.Error(t, err)
+
+	_, _, err = BasicAuthDecode("invalid")
+	assert.Error(t, err)
 }
 
 func TestBasicAuthEncode(t *testing.T) {

modules/context/api.go

@@ -255,3 +255,61 @@ func (ctx *APIContext) NotFound(objs ...interface{}) {
 		"errors":            errors,
 	})
 }
+
+// RepoRefForAPI handles repository reference names when the ref name is not explicitly given
+func RepoRefForAPI() macaron.Handler {
+	return func(ctx *APIContext) {
+		// Empty repository does not have reference information.
+		if ctx.Repo.Repository.IsEmpty {
+			return
+		}
+
+		var err error
+
+		if ctx.Repo.GitRepo == nil {
+			repoPath := models.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
+			ctx.Repo.GitRepo, err = git.OpenRepository(repoPath)
+			if err != nil {
+				ctx.InternalServerError(err)
+				return
+			}
+			// We opened it, we should close it
+			defer func() {
+				// If it's been set to nil then assume someone else has closed it.
+				if ctx.Repo.GitRepo != nil {
+					ctx.Repo.GitRepo.Close()
+				}
+			}()
+		}
+
+		refName := getRefName(ctx.Context, RepoRefAny)
+
+		if ctx.Repo.GitRepo.IsBranchExist(refName) {
+			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName)
+			if err != nil {
+				ctx.InternalServerError(err)
+				return
+			}
+			ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
+		} else if ctx.Repo.GitRepo.IsTagExist(refName) {
+			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName)
+			if err != nil {
+				ctx.InternalServerError(err)
+				return
+			}
+			ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
+		} else if len(refName) == 40 {
+			ctx.Repo.CommitID = refName
+			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refName)
+			if err != nil {
+				ctx.NotFound("GetCommit", err)
+				return
+			}
+		} else {
+			ctx.NotFound(fmt.Errorf("not exist: '%s'", ctx.Params("*")))
+			return
+		}
+
+		ctx.Next()
+	}
+}

modules/context/repo.go

@@ -704,7 +704,6 @@ func RepoRefByType(refType RepoRefType) macaron.Handler {
 			err     error
 		)
 
-		// For API calls.
 		if ctx.Repo.GitRepo == nil {
 			repoPath := models.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
 			ctx.Repo.GitRepo, err = git.OpenRepository(repoPath)
@@ -773,7 +772,7 @@ func RepoRefByType(refType RepoRefType) macaron.Handler {
 
 				ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refName)
 				if err != nil {
-					ctx.NotFound("GetCommit", nil)
+					ctx.NotFound("GetCommit", err)
 					return
 				}
 			} else {

modules/convert/pull_review.go

@@ -83,18 +83,17 @@ func ToPullReviewCommentList(review *models.Review, doer *models.User) ([]*api.P
 
 	apiComments := make([]*api.PullReviewComment, 0, len(review.CodeComments))
 
-	auth := false
-	if doer != nil {
-		auth = doer.IsAdmin || doer.ID == review.ReviewerID
-	}
-
 	for _, lines := range review.CodeComments {
 		for _, comments := range lines {
 			for _, comment := range comments {
+				auth := false
+				if doer != nil {
+					auth = doer.IsAdmin || doer.ID == comment.Poster.ID
+				}
 				apiComment := &api.PullReviewComment{
 					ID:           comment.ID,
 					Body:         comment.Content,
-					Reviewer:     ToUser(review.Reviewer, doer != nil, auth),
+					Reviewer:     ToUser(comment.Poster, doer != nil, auth),
 					ReviewID:     review.ID,
 					Created:      comment.CreatedUnix.AsTime(),
 					Updated:      comment.UpdatedUnix.AsTime(),

modules/convert/user.go

@@ -13,6 +13,10 @@ import (
 // ToUser convert models.User to api.User
 // signed shall only be set if requester is logged in. authed shall only be set if user is site admin or user himself
 func ToUser(user *models.User, signed, authed bool) *api.User {
+	if user == nil {
+		return nil
+	}
+
 	result := &api.User{
 		ID:        user.ID,
 		UserName:  user.Name,

modules/emoji/emoji.go

@@ -30,6 +30,9 @@ var (
 	// aliasMap provides a map of the alias to its emoji data.
 	aliasMap map[string]int
 
+	// emptyReplacer is the string replacer for emoji codes.
+	emptyReplacer *strings.Replacer
+
 	// codeReplacer is the string replacer for emoji codes.
 	codeReplacer *strings.Replacer
 
@@ -49,6 +52,7 @@ func loadMap() {
 
 		// process emoji codes and aliases
 		codePairs := make([]string, 0)
+		emptyPairs := make([]string, 0)
 		aliasPairs := make([]string, 0)
 
 		// sort from largest to small so we match combined emoji first
@@ -64,6 +68,7 @@ func loadMap() {
 			// setup codes
 			codeMap[e.Emoji] = i
 			codePairs = append(codePairs, e.Emoji, ":"+e.Aliases[0]+":")
+			emptyPairs = append(emptyPairs, e.Emoji, e.Emoji)
 
 			// setup aliases
 			for _, a := range e.Aliases {
@@ -77,6 +82,7 @@ func loadMap() {
 		}
 
 		// create replacers
+		emptyReplacer = strings.NewReplacer(emptyPairs...)
 		codeReplacer = strings.NewReplacer(codePairs...)
 		aliasReplacer = strings.NewReplacer(aliasPairs...)
 	})
@@ -127,38 +133,53 @@ func ReplaceAliases(s string) string {
 	return aliasReplacer.Replace(s)
 }
 
+type rememberSecondWriteWriter struct {
+	pos        int
+	idx        int
+	end        int
+	writecount int
+}
+
+func (n *rememberSecondWriteWriter) Write(p []byte) (int, error) {
+	n.writecount++
+	if n.writecount == 2 {
+		n.idx = n.pos
+		n.end = n.pos + len(p)
+	}
+	n.pos += len(p)
+	return len(p), nil
+}
+
+func (n *rememberSecondWriteWriter) WriteString(s string) (int, error) {
+	n.writecount++
+	if n.writecount == 2 {
+		n.idx = n.pos
+		n.end = n.pos + len(s)
+	}
+	n.pos += len(s)
+	return len(s), nil
+}
+
 // FindEmojiSubmatchIndex returns index pair of longest emoji in a string
 func FindEmojiSubmatchIndex(s string) []int {
 	loadMap()
-	found := make(map[int]int)
-	keys := make([]int, 0)
+	secondWriteWriter := rememberSecondWriteWriter{}
 
-	//see if there are any emoji in string before looking for position of specific ones
-	//no performance difference when there is a match but 10x faster when there are not
-	if s == ReplaceCodes(s) {
+	// A faster and clean implementation would copy the trie tree formation in strings.NewReplacer but
+	// we can be lazy here.
+	//
+	// The implementation of strings.Replacer.WriteString is such that the first index of the emoji
+	// submatch is simply the second thing that is written to WriteString in the writer.
+	//
+	// Therefore we can simply take the index of the second write as our first emoji
+	//
+	// FIXME: just copy the trie implementation from strings.NewReplacer
+	_, _ = emptyReplacer.WriteString(&secondWriteWriter, s)
+
+	// if we wrote less than twice then we never "replaced"
+	if secondWriteWriter.writecount < 2 {
 		return nil
 	}
 
-	// get index of first emoji occurrence while also checking for longest combination
-	for j := range GemojiData {
-		i := strings.Index(s, GemojiData[j].Emoji)
-		if i != -1 {
-			if _, ok := found[i]; !ok {
-				if len(keys) == 0 || i < keys[0] {
-					found[i] = j
-					keys = []int{i}
-				}
-				if i == 0 {
-					break
-				}
-			}
-		}
-	}
-
-	if len(keys) > 0 {
-		index := keys[0]
-		return []int{index, index + len(GemojiData[found[index]].Emoji)}
-	}
-
-	return nil
+	return []int{secondWriteWriter.idx, secondWriteWriter.end}
 }

modules/emoji/emoji_test.go

@@ -8,6 +8,8 @@ package emoji
 import (
 	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestDumpInfo(t *testing.T) {
@@ -65,3 +67,34 @@ func TestReplacers(t *testing.T) {
 		}
 	}
 }
+
+func TestFindEmojiSubmatchIndex(t *testing.T) {
+	type testcase struct {
+		teststring string
+		expected   []int
+	}
+
+	testcases := []testcase{
+		{
+			"\U0001f44d",
+			[]int{0, len("\U0001f44d")},
+		},
+		{
+			"\U0001f44d +1 \U0001f44d \U0001f37a",
+			[]int{0, 4},
+		},
+		{
+			" \U0001f44d",
+			[]int{1, 1 + len("\U0001f44d")},
+		},
+		{
+			string([]byte{'\u0001'}) + "\U0001f44d",
+			[]int{1, 1 + len("\U0001f44d")},
+		},
+	}
+
+	for _, kase := range testcases {
+		actual := FindEmojiSubmatchIndex(kase.teststring)
+		assert.Equal(t, kase.expected, actual)
+	}
+}

modules/git/blame.go

@@ -27,7 +27,7 @@ type BlameReader struct {
 	cmd     *exec.Cmd
 	pid     int64
 	output  io.ReadCloser
-	scanner *bufio.Scanner
+	reader  *bufio.Reader
 	lastSha *string
 	cancel  context.CancelFunc
 }
@@ -38,23 +38,30 @@ var shaLineRegex = regexp.MustCompile("^([a-z0-9]{40})")
 func (r *BlameReader) NextPart() (*BlamePart, error) {
 	var blamePart *BlamePart
 
-	scanner := r.scanner
+	reader := r.reader
 
 	if r.lastSha != nil {
 		blamePart = &BlamePart{*r.lastSha, make([]string, 0)}
 	}
 
-	for scanner.Scan() {
-		line := scanner.Text()
+	var line []byte
+	var isPrefix bool
+	var err error
+
+	for err != io.EOF {
+		line, isPrefix, err = reader.ReadLine()
+		if err != nil && err != io.EOF {
+			return blamePart, err
+		}
 
-		// Skip empty lines
 		if len(line) == 0 {
+			// isPrefix will be false
 			continue
 		}
 
-		lines := shaLineRegex.FindStringSubmatch(line)
+		lines := shaLineRegex.FindSubmatch(line)
 		if lines != nil {
-			sha1 := lines[1]
+			sha1 := string(lines[1])
 
 			if blamePart == nil {
 				blamePart = &BlamePart{sha1, make([]string, 0)}
@@ -62,12 +69,27 @@ func (r *BlameReader) NextPart() (*BlamePart, error) {
 
 			if blamePart.Sha != sha1 {
 				r.lastSha = &sha1
+				// need to munch to end of line...
+				for isPrefix {
+					_, isPrefix, err = reader.ReadLine()
+					if err != nil && err != io.EOF {
+						return blamePart, err
+					}
+				}
 				return blamePart, nil
 			}
 		} else if line[0] == '\t' {
 			code := line[1:]
 
-			blamePart.Lines = append(blamePart.Lines, code)
+			blamePart.Lines = append(blamePart.Lines, string(code))
+		}
+
+		// need to munch to end of line...
+		for isPrefix {
+			_, isPrefix, err = reader.ReadLine()
+			if err != nil && err != io.EOF {
+				return blamePart, err
+			}
 		}
 	}
 
@@ -121,13 +143,13 @@ func createBlameReader(ctx context.Context, dir string, command ...string) (*Bla
 
 	pid := process.GetManager().Add(fmt.Sprintf("GetBlame [repo_path: %s]", dir), cancel)
 
-	scanner := bufio.NewScanner(stdout)
+	reader := bufio.NewReader(stdout)
 
 	return &BlameReader{
 		cmd,
 		pid,
 		stdout,
-		scanner,
+		reader,
 		nil,
 		cancel,
 	}, nil

modules/git/command.go

@@ -153,6 +153,7 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
 		err := fn(ctx, cancel)
 		if err != nil {
 			cancel()
+			_ = cmd.Wait()
 			return err
 		}
 	}

modules/git/commit.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
+	"errors"
 	"fmt"
 	"image"
 	"image/color"
@@ -17,6 +18,7 @@ import (
 	_ "image/png"  // for processing png images
 	"io"
 	"net/http"
+	"os/exec"
 	"strconv"
 	"strings"
 
@@ -309,23 +311,33 @@ func (c *Commit) CommitsBefore() (*list.List, error) {
 
 // HasPreviousCommit returns true if a given commitHash is contained in commit's parents
 func (c *Commit) HasPreviousCommit(commitHash SHA1) (bool, error) {
-	for i := 0; i < c.ParentCount(); i++ {
-		commit, err := c.Parent(i)
-		if err != nil {
-			return false, err
-		}
-		if commit.ID == commitHash {
-			return true, nil
-		}
-		commitInParentCommit, err := commit.HasPreviousCommit(commitHash)
-		if err != nil {
-			return false, err
-		}
-		if commitInParentCommit {
-			return true, nil
-		}
+	this := c.ID.String()
+	that := commitHash.String()
+
+	if this == that {
+		return false, nil
 	}
-	return false, nil
+
+	if err := CheckGitVersionConstraint(">= 1.8.0"); err == nil {
+		_, err := NewCommand("merge-base", "--is-ancestor", that, this).RunInDir(c.repo.Path)
+		if err == nil {
+			return true, nil
+		}
+		var exitError *exec.ExitError
+		if errors.As(err, &exitError) {
+			if exitError.ProcessState.ExitCode() == 1 && len(exitError.Stderr) == 0 {
+				return false, nil
+			}
+		}
+		return false, err
+	}
+
+	result, err := NewCommand("rev-list", "--ancestry-path", "-n1", that+".."+this, "--").RunInDir(c.repo.Path)
+	if err != nil {
+		return false, err
+	}
+
+	return len(strings.TrimSpace(result)) > 0, nil
 }
 
 // CommitsBeforeLimit returns num commits before current revision

modules/git/diff.go

@@ -47,7 +47,7 @@ func GetRawDiffForFile(repoPath, startCommit, endCommit string, diffType RawDiff
 func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diffType RawDiffType, file string, writer io.Writer) error {
 	commit, err := repo.GetCommit(endCommit)
 	if err != nil {
-		return fmt.Errorf("GetCommit: %v", err)
+		return err
 	}
 	fileArgs := make([]string, 0)
 	if len(file) > 0 {
@@ -125,30 +125,39 @@ var hunkRegex = regexp.MustCompile(`^@@ -(?P<beginOld>[0-9]+)(,(?P<endOld>[0-9]+
 
 const cmdDiffHead = "diff --git "
 
-func isHeader(lof string) bool {
-	return strings.HasPrefix(lof, cmdDiffHead) || strings.HasPrefix(lof, "---") || strings.HasPrefix(lof, "+++")
+func isHeader(lof string, inHunk bool) bool {
+	return strings.HasPrefix(lof, cmdDiffHead) || (!inHunk && (strings.HasPrefix(lof, "---") || strings.HasPrefix(lof, "+++")))
 }
 
 // CutDiffAroundLine cuts a diff of a file in way that only the given line + numberOfLine above it will be shown
 // it also recalculates hunks and adds the appropriate headers to the new diff.
 // Warning: Only one-file diffs are allowed.
-func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLine int) string {
+func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLine int) (string, error) {
 	if line == 0 || numbersOfLine == 0 {
 		// no line or num of lines => no diff
-		return ""
+		return "", nil
 	}
+
 	scanner := bufio.NewScanner(originalDiff)
 	hunk := make([]string, 0)
+
 	// begin is the start of the hunk containing searched line
 	// end is the end of the hunk ...
 	// currentLine is the line number on the side of the searched line (differentiated by old)
 	// otherLine is the line number on the opposite side of the searched line (differentiated by old)
 	var begin, end, currentLine, otherLine int64
 	var headerLines int
+
+	inHunk := false
+
 	for scanner.Scan() {
 		lof := scanner.Text()
 		// Add header to enable parsing
-		if isHeader(lof) {
+
+		if isHeader(lof, inHunk) {
+			if strings.HasPrefix(lof, cmdDiffHead) {
+				inHunk = false
+			}
 			hunk = append(hunk, lof)
 			headerLines++
 		}
@@ -157,6 +166,7 @@ func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLi
 		}
 		// Detect "hunk" with contains commented lof
 		if strings.HasPrefix(lof, "@@") {
+			inHunk = true
 			// Already got our hunk. End of hunk detected!
 			if len(hunk) > headerLines {
 				break
@@ -213,15 +223,19 @@ func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLi
 			}
 		}
 	}
+	err := scanner.Err()
+	if err != nil {
+		return "", err
+	}
 
 	// No hunk found
 	if currentLine == 0 {
-		return ""
+		return "", nil
 	}
 	// headerLines + hunkLine (1) = totalNonCodeLines
 	if len(hunk)-headerLines-1 <= numbersOfLine {
 		// No need to cut the hunk => return existing hunk
-		return strings.Join(hunk, "\n")
+		return strings.Join(hunk, "\n"), nil
 	}
 	var oldBegin, oldNumOfLines, newBegin, newNumOfLines int64
 	if old {
@@ -256,5 +270,5 @@ func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLi
 	// construct the new hunk header
 	newHunk[headerLines] = fmt.Sprintf("@@ -%d,%d +%d,%d @@",
 		oldBegin, oldNumOfLines, newBegin, newNumOfLines)
-	return strings.Join(newHunk, "\n")
+	return strings.Join(newHunk, "\n"), nil
 }

modules/git/diff_test.go

@@ -23,8 +23,28 @@ const exampleDiff = `diff --git a/README.md b/README.md
 + cut off
 + cut off`
 
+const breakingDiff = `diff --git a/aaa.sql b/aaa.sql
+index d8e4c92..19dc8ad 100644
+--- a/aaa.sql
++++ b/aaa.sql
+@@ -1,9 +1,10 @@
+ --some comment
+--- some comment 5
++--some coment 2
++-- some comment 3
+ create or replace procedure test(p1 varchar2)
+ is
+ begin
+---new comment
+ dbms_output.put_line(p1);
++--some other comment
+ end;
+ /
+`
+
 func TestCutDiffAroundLine(t *testing.T) {
-	result := CutDiffAroundLine(strings.NewReader(exampleDiff), 4, false, 3)
+	result, err := CutDiffAroundLine(strings.NewReader(exampleDiff), 4, false, 3)
+	assert.NoError(t, err)
 	resultByLine := strings.Split(result, "\n")
 	assert.Len(t, resultByLine, 7)
 	// Check if headers got transferred
@@ -37,18 +57,50 @@ func TestCutDiffAroundLine(t *testing.T) {
 	assert.Equal(t, "+ Build Status", resultByLine[4])
 
 	// Must be same result as before since old line 3 == new line 5
-	newResult := CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3)
+	newResult, err := CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3)
+	assert.NoError(t, err)
 	assert.Equal(t, result, newResult, "Must be same result as before since old line 3 == new line 5")
 
-	newResult = CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 300)
+	newResult, err = CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 300)
+	assert.NoError(t, err)
 	assert.Equal(t, exampleDiff, newResult)
 
-	emptyResult := CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 0)
+	emptyResult, err := CutDiffAroundLine(strings.NewReader(exampleDiff), 6, false, 0)
+	assert.NoError(t, err)
 	assert.Empty(t, emptyResult)
 
 	// Line is out of scope
-	emptyResult = CutDiffAroundLine(strings.NewReader(exampleDiff), 434, false, 0)
+	emptyResult, err = CutDiffAroundLine(strings.NewReader(exampleDiff), 434, false, 0)
+	assert.NoError(t, err)
 	assert.Empty(t, emptyResult)
+
+	// Handle minus diffs properly
+	minusDiff, err := CutDiffAroundLine(strings.NewReader(breakingDiff), 2, false, 4)
+	assert.NoError(t, err)
+
+	expected := `diff --git a/aaa.sql b/aaa.sql
+--- a/aaa.sql
++++ b/aaa.sql
+@@ -1,9 +1,10 @@
+ --some comment
+--- some comment 5
++--some coment 2`
+	assert.Equal(t, expected, minusDiff)
+
+	// Handle minus diffs properly
+	minusDiff, err = CutDiffAroundLine(strings.NewReader(breakingDiff), 3, false, 4)
+	assert.NoError(t, err)
+
+	expected = `diff --git a/aaa.sql b/aaa.sql
+--- a/aaa.sql
++++ b/aaa.sql
+@@ -1,9 +1,10 @@
+ --some comment
+--- some comment 5
++--some coment 2
++-- some comment 3`
+
+	assert.Equal(t, expected, minusDiff)
 }
 
 func BenchmarkCutDiffAroundLine(b *testing.B) {
@@ -69,7 +121,7 @@ func ExampleCutDiffAroundLine() {
  Docker Pulls
 + cut off
 + cut off`
-	result := CutDiffAroundLine(strings.NewReader(diff), 4, false, 3)
+	result, _ := CutDiffAroundLine(strings.NewReader(diff), 4, false, 3)
 	println(result)
 }
 

modules/git/git.go

@@ -32,6 +32,7 @@ var (
 	GitExecutable = "git"
 
 	// DefaultContext is the default context to run git commands in
+	// will be overwritten by Init with HammerContext
 	DefaultContext = context.Background()
 
 	gitVersion *version.Version

modules/git/repo.go

@@ -8,6 +8,7 @@ package git
 import (
 	"bytes"
 	"container/list"
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -166,19 +167,24 @@ type CloneRepoOptions struct {
 
 // Clone clones original repository to target path.
 func Clone(from, to string, opts CloneRepoOptions) (err error) {
+	return CloneWithContext(DefaultContext, from, to, opts)
+}
+
+// CloneWithContext clones original repository to target path.
+func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
 	cargs := make([]string, len(GlobalCommandArgs))
 	copy(cargs, GlobalCommandArgs)
-	return CloneWithArgs(from, to, cargs, opts)
+	return CloneWithArgs(ctx, from, to, cargs, opts)
 }
 
 // CloneWithArgs original repository to target path.
-func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err error) {
+func CloneWithArgs(ctx context.Context, from, to string, args []string, opts CloneRepoOptions) (err error) {
 	toDir := path.Dir(to)
 	if err = os.MkdirAll(toDir, os.ModePerm); err != nil {
 		return err
 	}
 
-	cmd := NewCommandNoGlobals(args...).AddArguments("clone")
+	cmd := NewCommandContextNoGlobals(ctx, args...).AddArguments("clone")
 	if opts.Mirror {
 		cmd.AddArguments("--mirror")
 	}

modules/git/repo_commit.go

@@ -9,6 +9,8 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strconv"
 	"strings"
 
@@ -45,14 +47,7 @@ func (repo *Repository) GetBranchCommitID(name string) (string, error) {
 
 // GetTagCommitID returns last commit ID string of given tag.
 func (repo *Repository) GetTagCommitID(name string) (string, error) {
-	stdout, err := NewCommand("rev-list", "-n", "1", TagPrefix+name).RunInDir(repo.Path)
-	if err != nil {
-		if strings.Contains(err.Error(), "unknown revision or path") {
-			return "", ErrNotExist{name, ""}
-		}
-		return "", err
-	}
-	return strings.TrimSpace(stdout), nil
+	return repo.GetRefCommitID(TagPrefix + name)
 }
 
 func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
@@ -129,19 +124,23 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
 
 // ConvertToSHA1 returns a Hash object from a potential ID string
 func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
-	if len(commitID) != 40 {
-		var err error
-		actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path)
-		if err != nil {
-			if strings.Contains(err.Error(), "unknown revision or path") ||
-				strings.Contains(err.Error(), "fatal: Needed a single revision") {
-				return SHA1{}, ErrNotExist{commitID, ""}
-			}
-			return SHA1{}, err
+	if len(commitID) == 40 {
+		sha1, err := NewIDFromString(commitID)
+		if err == nil {
+			return sha1, nil
 		}
-		commitID = actualCommitID
 	}
-	return NewIDFromString(commitID)
+
+	actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path)
+	if err != nil {
+		if strings.Contains(err.Error(), "unknown revision or path") ||
+			strings.Contains(err.Error(), "fatal: Needed a single revision") {
+			return SHA1{}, ErrNotExist{commitID, ""}
+		}
+		return SHA1{}, err
+	}
+
+	return NewIDFromString(actualCommitID)
 }
 
 // GetCommit returns commit object of by ID string.
@@ -323,8 +322,41 @@ func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) {
 
 // CommitsByFileAndRange return the commits according revison file and the page
 func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (*list.List, error) {
-	stdout, err := NewCommand("log", revision, "--follow", "--skip="+strconv.Itoa((page-1)*50),
-		"--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path)
+	skip := (page - 1) * CommitsRangeSize
+
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+	go func() {
+		stderr := strings.Builder{}
+		err := NewCommand("log", revision, "--follow",
+			"--max-count="+strconv.Itoa(CommitsRangeSize*page),
+			prettyLogFormat, "--", file).
+			RunInDirPipeline(repo.Path, stdoutWriter, &stderr)
+		if err != nil {
+			if stderr.Len() > 0 {
+				err = fmt.Errorf("%v - %s", err, stderr.String())
+			}
+			_ = stdoutWriter.CloseWithError(err)
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	if skip > 0 {
+		_, err := io.CopyN(ioutil.Discard, stdoutReader, int64(skip*41))
+		if err != nil {
+			if err == io.EOF {
+				return list.New(), nil
+			}
+			_ = stdoutReader.CloseWithError(err)
+			return nil, err
+		}
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutReader)
 	if err != nil {
 		return nil, err
 	}

modules/git/repo_language_stats.go

@@ -44,7 +44,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 	sizes := make(map[string]int64)
 	err = tree.Files().ForEach(func(f *object.File) error {
-		if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
+		if f.Size == 0 || analyze.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
 			enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
 			return nil
 		}

modules/highlight/highlight.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync"
 
+	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"github.com/alecthomas/chroma/formatters/html"
@@ -117,9 +118,11 @@ func File(numLines int, fileName string, code []byte) map[int]string {
 		fileName = "test." + val
 	}
 
-	lexer := lexers.Match(fileName)
+	language := analyze.GetCodeLanguage(fileName, code)
+
+	lexer := lexers.Get(language)
 	if lexer == nil {
-		lexer = lexers.Analyse(string(code))
+		lexer = lexers.Match(fileName)
 		if lexer == nil {
 			lexer = lexers.Fallback
 		}

modules/indexer/code/bleve.go

@@ -175,7 +175,7 @@ func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) {
 
 func (b *BleveIndexer) addUpdate(commitSha string, update fileUpdate, repo *models.Repository, batch rupture.FlushingBatch) error {
 	// Ignore vendored files in code search
-	if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
+	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
 		return nil
 	}
 

modules/indexer/code/elastic_search.go

@@ -170,7 +170,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
 
 func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) {
 	// Ignore vendored files in code search
-	if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
+	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
 		return nil, nil
 	}
 

modules/lfs/content_store.go

@@ -8,6 +8,8 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"fmt"
+	"hash"
 	"io"
 	"os"
 
@@ -21,6 +23,21 @@ var (
 	errSizeMismatch = errors.New("Content size does not match")
 )
 
+// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
+type ErrRangeNotSatisfiable struct {
+	FromByte int64
+}
+
+func (err ErrRangeNotSatisfiable) Error() string {
+	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
+}
+
+// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
+func IsErrRangeNotSatisfiable(err error) bool {
+	_, ok := err.(ErrRangeNotSatisfiable)
+	return ok
+}
+
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
 	storage.ObjectStorage
@@ -35,7 +52,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 		return nil, err
 	}
 	if fromByte > 0 {
-		_, err = f.Seek(fromByte, os.SEEK_CUR)
+		if fromByte >= meta.Size {
+			return nil, ErrRangeNotSatisfiable{
+				FromByte: fromByte,
+			}
+		}
+		_, err = f.Seek(fromByte, io.SeekStart)
 		if err != nil {
 			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
 		}
@@ -45,15 +67,20 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
 func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
-	hash := sha256.New()
-	rd := io.TeeReader(r, hash)
 	p := meta.RelativePath()
-	written, err := s.Save(p, rd)
+
+	// Wrap the provided reader with an inline hashing and size checker
+	wrappedRd := newHashingReader(meta.Size, meta.Oid, r)
+
+	// now pass the wrapped reader to Save - if there is a size mismatch or hash mismatch then
+	// the errors returned by the newHashingReader should percolate up to here
+	written, err := s.Save(p, wrappedRd, meta.Size)
 	if err != nil {
 		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
 		return err
 	}
 
+	// This shouldn't happen but it is sensible to test
 	if written != meta.Size {
 		if err := s.Delete(p); err != nil {
 			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
@@ -61,14 +88,6 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
 		return errSizeMismatch
 	}
 
-	shaStr := hex.EncodeToString(hash.Sum(nil))
-	if shaStr != meta.Oid {
-		if err := s.Delete(p); err != nil {
-			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
-		}
-		return errHashMismatch
-	}
-
 	return nil
 }
 
@@ -97,3 +116,45 @@ func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
 
 	return true, nil
 }
+
+type hashingReader struct {
+	internal     io.Reader
+	currentSize  int64
+	expectedSize int64
+	hash         hash.Hash
+	expectedHash string
+}
+
+func (r *hashingReader) Read(b []byte) (int, error) {
+	n, err := r.internal.Read(b)
+
+	if n > 0 {
+		r.currentSize += int64(n)
+		wn, werr := r.hash.Write(b[:n])
+		if wn != n || werr != nil {
+			return n, werr
+		}
+	}
+
+	if err != nil && err == io.EOF {
+		if r.currentSize != r.expectedSize {
+			return n, errSizeMismatch
+		}
+
+		shaStr := hex.EncodeToString(r.hash.Sum(nil))
+		if shaStr != r.expectedHash {
+			return n, errHashMismatch
+		}
+	}
+
+	return n, err
+}
+
+func newHashingReader(expectedSize int64, expectedHash string, reader io.Reader) *hashingReader {
+	return &hashingReader{
+		internal:     reader,
+		expectedSize: expectedSize,
+		expectedHash: expectedHash,
+		hash:         sha256.New(),
+	}
+}

modules/lfs/server.go

@@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
 	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	content, err := contentStore.Get(meta, fromByte)
 	if err != nil {
-		// Errors are logged in contentStore.Get
-		writeStatus(ctx, 404)
+		if IsErrRangeNotSatisfiable(err) {
+			writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
+		} else {
+			// Errors are logged in contentStore.Get
+			writeStatus(ctx, 404)
+		}
 		return
 	}
 	defer content.Close()

modules/markup/html.go

@@ -43,7 +43,7 @@ var (
 	// sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae
 	// Although SHA1 hashes are 40 chars long, the regex matches the hash from 7 to 40 chars in length
 	// so that abbreviated hash links can be used as well. This matches git and github useability.
-	sha1CurrentPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-f]{7,40})(?:\s|$|\)|\]|\.(\s|$))`)
+	sha1CurrentPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-f]{7,40})(?:\s|$|\)|\]|[.,](\s|$))`)
 
 	// shortLinkPattern matches short but difficult to parse [[name|link|arg=test]] syntax
 	shortLinkPattern = regexp.MustCompile(`\[\[(.*?)\]\](\w*)`)
@@ -298,8 +298,8 @@ func RenderEmoji(
 	return ctx.postProcess(rawHTML)
 }
 
-var byteBodyTag = []byte("<body>")
-var byteBodyTagClosing = []byte("</body>")
+var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`)
+var nulCleaner = strings.NewReplacer("\000", "")
 
 func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
 	if ctx.procs == nil {
@@ -307,13 +307,18 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
 	}
 
 	// give a generous extra 50 bytes
-	res := make([]byte, 0, len(rawHTML)+50)
-	res = append(res, byteBodyTag...)
-	res = append(res, rawHTML...)
-	res = append(res, byteBodyTagClosing...)
+	res := bytes.NewBuffer(make([]byte, 0, len(rawHTML)+50))
+	// prepend "<html><body>"
+	_, _ = res.WriteString("<html><body>")
+
+	// Strip out nuls - they're always invalid
+	_, _ = res.Write(tagCleaner.ReplaceAll([]byte(nulCleaner.Replace(string(rawHTML))), []byte("&lt;$1")))
+
+	// close the tags
+	_, _ = res.WriteString("</body></html>")
 
 	// parse the HTML
-	nodes, err := html.ParseFragment(bytes.NewReader(res), nil)
+	nodes, err := html.ParseFragment(res, nil)
 	if err != nil {
 		return nil, &postProcessError{"invalid HTML", err}
 	}
@@ -322,24 +327,45 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
 		ctx.visitNode(node, true)
 	}
 
+	newNodes := make([]*html.Node, 0, len(nodes))
+
+	for _, node := range nodes {
+		if node.Data == "html" {
+			node = node.FirstChild
+			for node != nil && node.Data != "body" {
+				node = node.NextSibling
+			}
+		}
+		if node == nil {
+			continue
+		}
+		if node.Data == "body" {
+			child := node.FirstChild
+			for child != nil {
+				newNodes = append(newNodes, child)
+				child = child.NextSibling
+			}
+		} else {
+			newNodes = append(newNodes, node)
+		}
+	}
+
+	nodes = newNodes
+
 	// Create buffer in which the data will be placed again. We know that the
 	// length will be at least that of res; to spare a few alloc+copy, we
 	// reuse res, resetting its length to 0.
-	buf := bytes.NewBuffer(res[:0])
+	res.Reset()
 	// Render everything to buf.
 	for _, node := range nodes {
-		err = html.Render(buf, node)
+		err = html.Render(res, node)
 		if err != nil {
 			return nil, &postProcessError{"error rendering processed HTML", err}
 		}
 	}
 
-	// remove initial parts - because Render creates a whole HTML page.
-	res = buf.Bytes()
-	res = res[bytes.Index(res, byteBodyTag)+len(byteBodyTag) : bytes.LastIndex(res, byteBodyTagClosing)]
-
 	// Everything done successfully, return parsed data.
-	return res, nil
+	return res.Bytes(), nil
 }
 
 func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
@@ -632,16 +658,18 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 			// When parsing HTML, x/net/html will change all quotes which are
 			// not used for syntax into UTF-8 quotes. So checking val[0] won't
 			// be enough, since that only checks a single byte.
-			if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) ||
-				(strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) {
-				const lenQuote = len("‘")
-				val = val[lenQuote : len(val)-lenQuote]
-			} else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) ||
-				(strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) {
-				val = val[1 : len(val)-1]
-			} else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "’") {
-				const lenQuote = len("‘")
-				val = val[1 : len(val)-lenQuote]
+			if len(val) > 1 {
+				if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) ||
+					(strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) {
+					const lenQuote = len("‘")
+					val = val[lenQuote : len(val)-lenQuote]
+				} else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) ||
+					(strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) {
+					val = val[1 : len(val)-1]
+				} else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "’") {
+					const lenQuote = len("‘")
+					val = val[1 : len(val)-lenQuote]
+				}
 			}
 			props[key] = val
 		}

modules/markup/html_test.go

@@ -46,6 +46,12 @@ func TestRender_Commits(t *testing.T) {
 	test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>")
 	test("deadbeef", `<p>deadbeef</p>`)
 	test("d27ace93", `<p>d27ace93</p>`)
+	test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`)
+
+	expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>`
+	test(sha[:14]+".", `<p>`+expected14+`.</p>`)
+	test(sha[:14]+",", `<p>`+expected14+`,</p>`)
+	test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`)
 }
 
 func TestRender_CrossReferences(t *testing.T) {
@@ -142,7 +148,7 @@ func TestRender_links(t *testing.T) {
 		`<p><a href="ftp://gitea.com/file.txt" rel="nofollow">ftp://gitea.com/file.txt</a></p>`)
 	test(
 		"magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download",
-		`<p><a href="magnet:?dn=download&xt=urn%3Abtih%3A5dee65101db281ac9c46344cd6b175cdcadabcde" rel="nofollow">magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&amp;dn=download</a></p>`)
+		`<p><a href="magnet:?xt=urn%3Abtih%3A5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download" rel="nofollow">magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&amp;dn=download</a></p>`)
 
 	// Test that should *not* be turned into URL
 	test(
@@ -377,3 +383,28 @@ func TestRender_ShortLinks(t *testing.T) {
 		`<p><a href="https://example.org" rel="nofollow">[[foobar]]</a></p>`,
 		`<p><a href="https://example.org" rel="nofollow">[[foobar]]</a></p>`)
 }
+
+func Test_ParseClusterFuzz(t *testing.T) {
+	setting.AppURL = AppURL
+	setting.AppSubURL = AppSubURL
+
+	var localMetas = map[string]string{
+		"user": "go-gitea",
+		"repo": "gitea",
+	}
+
+	data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
+
+	val, err := PostProcess([]byte(data), "https://example.com", localMetas, false)
+
+	assert.NoError(t, err)
+	assert.NotContains(t, string(val), "<html")
+
+	data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
+
+	val, err = PostProcess([]byte(data), "https://example.com", localMetas, false)
+
+	assert.NoError(t, err)
+
+	assert.NotContains(t, string(val), "<html")
+}

modules/markup/markdown/goldmark.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"strings"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/common"
 	"code.gitea.io/gitea/modules/setting"
@@ -76,6 +77,12 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
 					header.ID = util.BytesToReadOnlyString(id.([]byte))
 				}
 				toc = append(toc, header)
+			} else {
+				for _, attr := range v.Attributes() {
+					if _, ok := attr.Value.([]byte); !ok {
+						v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value)))
+					}
+				}
 			}
 		case *ast.Image:
 			// Images need two things:
@@ -101,11 +108,41 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
 			parent := n.Parent()
 			// Create a link around image only if parent is not already a link
 			if _, ok := parent.(*ast.Link); !ok && parent != nil {
+				next := n.NextSibling()
+
+				// Create a link wrapper
 				wrap := ast.NewLink()
 				wrap.Destination = link
 				wrap.Title = v.Title
+
+				// Duplicate the current image node
+				image := ast.NewImage(ast.NewLink())
+				image.Destination = link
+				image.Title = v.Title
+				for _, attr := range v.Attributes() {
+					image.SetAttribute(attr.Name, attr.Value)
+				}
+				for child := v.FirstChild(); child != nil; {
+					next := child.NextSibling()
+					image.AppendChild(image, child)
+					child = next
+				}
+
+				// Append our duplicate image to the wrapper link
+				wrap.AppendChild(wrap, image)
+
+				// Wire in the next sibling
+				wrap.SetNextSibling(next)
+
+				// Replace the current node with the wrapper link
 				parent.ReplaceChild(parent, n, wrap)
-				wrap.AppendChild(wrap, n)
+
+				// But most importantly ensure the next sibling is still on the old image too
+				v.SetNextSibling(next)
+
+			} else {
+				log.Debug("ast.Image: %s has parent: %v", link, parent)
+
 			}
 		case *ast.Link:
 			// Links need their href to munged to be a real value

modules/markup/markdown/markdown.go

@@ -6,7 +6,8 @@
 package markdown
 
 import (
-	"bytes"
+	"fmt"
+	"io"
 	"strings"
 	"sync"
 
@@ -18,7 +19,7 @@ import (
 
 	chromahtml "github.com/alecthomas/chroma/formatters/html"
 	"github.com/yuin/goldmark"
-	"github.com/yuin/goldmark-highlighting"
+	highlighting "github.com/yuin/goldmark-highlighting"
 	meta "github.com/yuin/goldmark-meta"
 	"github.com/yuin/goldmark/extension"
 	"github.com/yuin/goldmark/parser"
@@ -34,6 +35,44 @@ var urlPrefixKey = parser.NewContextKey()
 var isWikiKey = parser.NewContextKey()
 var renderMetasKey = parser.NewContextKey()
 
+type closesWithError interface {
+	io.WriteCloser
+	CloseWithError(err error) error
+}
+
+type limitWriter struct {
+	w     closesWithError
+	sum   int64
+	limit int64
+}
+
+// Write implements the standard Write interface:
+func (l *limitWriter) Write(data []byte) (int, error) {
+	leftToWrite := l.limit - l.sum
+	if leftToWrite < int64(len(data)) {
+		n, err := l.w.Write(data[:leftToWrite])
+		l.sum += int64(n)
+		if err != nil {
+			return n, err
+		}
+		_ = l.w.Close()
+		return n, fmt.Errorf("Rendered content too large - truncating render")
+	}
+	n, err := l.w.Write(data)
+	l.sum += int64(n)
+	return n, err
+}
+
+// Close closes the writer
+func (l *limitWriter) Close() error {
+	return l.w.Close()
+}
+
+// CloseWithError closes the writer
+func (l *limitWriter) CloseWithError(err error) error {
+	return l.w.CloseWithError(err)
+}
+
 // NewGiteaParseContext creates a parser.Context with the gitea context set
 func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool) parser.Context {
 	pc := parser.NewContext(parser.WithIDs(newPrefixedIDs()))
@@ -43,8 +82,8 @@ func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool
 	return pc
 }
 
-// render renders Markdown to HTML without handling special links.
-func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte {
+// actualRender renders Markdown to HTML without handling special links.
+func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte {
 	once.Do(func() {
 		converter = goldmark.New(
 			goldmark.WithExtensions(extension.Table,
@@ -119,12 +158,57 @@ func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown
 
 	})
 
-	pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown)
-	var buf bytes.Buffer
-	if err := converter.Convert(giteautil.NormalizeEOL(body), &buf, parser.WithContext(pc)); err != nil {
-		log.Error("Unable to render: %v", err)
+	rd, wr := io.Pipe()
+	defer func() {
+		_ = rd.Close()
+		_ = wr.Close()
+	}()
+
+	lw := &limitWriter{
+		w:     wr,
+		limit: setting.UI.MaxDisplayFileSize * 3,
 	}
-	return markup.SanitizeReader(&buf).Bytes()
+
+	// FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long?
+	go func() {
+		defer func() {
+			err := recover()
+			if err == nil {
+				return
+			}
+
+			log.Warn("Unable to render markdown due to panic in goldmark: %v", err)
+			if log.IsDebug() {
+				log.Debug("Panic in markdown: %v\n%s", err, string(log.Stack(2)))
+			}
+			_ = lw.CloseWithError(fmt.Errorf("%v", err))
+		}()
+
+		pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown)
+		if err := converter.Convert(giteautil.NormalizeEOL(body), lw, parser.WithContext(pc)); err != nil {
+			log.Error("Unable to render: %v", err)
+			_ = lw.CloseWithError(err)
+			return
+		}
+		_ = lw.Close()
+	}()
+	return markup.SanitizeReader(rd).Bytes()
+}
+
+func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) (ret []byte) {
+	defer func() {
+		err := recover()
+		if err == nil {
+			return
+		}
+
+		log.Warn("Unable to render markdown due to panic in goldmark - will return sanitized raw bytes")
+		if log.IsDebug() {
+			log.Debug("Panic in markdown: %v\n%s", err, string(log.Stack(2)))
+		}
+		ret = markup.SanitizeBytes(body)
+	}()
+	return actualRender(body, urlPrefix, metas, wikiMarkdown)
 }
 
 var (

modules/markup/markdown/markdown_test.go

@@ -308,3 +308,34 @@ func TestRender_RenderParagraphs(t *testing.T) {
 	test(t, "A\n\nB\nC\n", 2)
 	test(t, "A\n\n\nB\nC\n", 2)
 }
+
+func TestMarkdownRenderRaw(t *testing.T) {
+	testcases := [][]byte{
+		{ // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6267570554535936
+			0x2a, 0x20, 0x2d, 0x0a, 0x09, 0x20, 0x60, 0x5b, 0x0a, 0x09, 0x20, 0x60,
+			0x5b,
+		},
+		{ // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6278827345051648
+			0x2d, 0x20, 0x2d, 0x0d, 0x09, 0x60, 0x0d, 0x09, 0x60,
+		},
+		{ // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6016973788020736[] = {
+			0x7b, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x35, 0x7d, 0x0a, 0x3d,
+		},
+	}
+
+	for _, testcase := range testcases {
+		_ = RenderRaw(testcase, "", false)
+	}
+}
+
+func TestRenderSiblingImages_Issue12925(t *testing.T) {
+	testcase := `![image1](/image1)
+![image2](/image2)
+`
+	expected := `<p><a href="/image1" rel="nofollow"><img src="/image1" alt="image1"></a><br>
+<a href="/image2" rel="nofollow"><img src="/image2" alt="image2"></a></p>
+`
+	res := string(RenderRaw([]byte(testcase), "", false))
+	assert.Equal(t, expected, res)
+
+}

modules/markup/sanitizer.go

@@ -46,7 +46,9 @@ func ReplaceSanitizer() {
 	sanitizer.policy.AllowAttrs("checked", "disabled", "readonly").OnElements("input")
 
 	// Custom URL-Schemes
-	sanitizer.policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
+	if len(setting.Markdown.CustomURLSchemes) > 0 {
+		sanitizer.policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
+	}
 
 	// Allow keyword markup
 	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^` + keywordClass + `$`)).OnElements("span")

modules/markup/sanitizer_test.go

@@ -6,6 +6,8 @@
 package markup
 
 import (
+	"html/template"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -50,3 +52,13 @@ func Test_Sanitizer(t *testing.T) {
 		assert.Equal(t, testCases[i+1], string(SanitizeBytes([]byte(testCases[i]))))
 	}
 }
+
+func TestSanitizeNonEscape(t *testing.T) {
+	descStr := "<scrİpt>&lt;script&gt;alert(document.domain)&lt;/script&gt;</scrİpt>"
+
+	output := template.HTML(Sanitize(string(descStr)))
+	if strings.Contains(string(output), "<script>") {
+		t.Errorf("un-escaped <script> in output: %q", output)
+	}
+
+}

modules/matchlist/matchlist.go

@@ -0,0 +1,46 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package matchlist
+
+import (
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+// Matchlist represents a block or allow list
+type Matchlist struct {
+	ruleGlobs []glob.Glob
+}
+
+// NewMatchlist creates a new block or allow list
+func NewMatchlist(rules ...string) (*Matchlist, error) {
+	for i := range rules {
+		rules[i] = strings.ToLower(rules[i])
+	}
+	list := Matchlist{
+		ruleGlobs: make([]glob.Glob, 0, len(rules)),
+	}
+
+	for _, rule := range rules {
+		rg, err := glob.Compile(rule)
+		if err != nil {
+			return nil, err
+		}
+		list.ruleGlobs = append(list.ruleGlobs, rg)
+	}
+
+	return &list, nil
+}
+
+// Match will matches
+func (b *Matchlist) Match(u string) bool {
+	for _, r := range b.ruleGlobs {
+		if r.Match(u) {
+			return true
+		}
+	}
+	return false
+}

modules/migrations/gitea_downloader.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
 	"code.gitea.io/gitea/modules/structs"
@@ -47,7 +48,7 @@ func (f *GiteaDownloaderFactory) New(ctx context.Context, opts base.MigrateOptio
 
 	path := strings.Split(repoNameSpace, "/")
 	if len(path) < 2 {
-		return nil, fmt.Errorf("invalid path")
+		return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
 	}
 
 	repoPath := strings.Join(path[len(path)-2:], "/")
@@ -87,7 +88,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
 		gitea_sdk.SetContext(ctx),
 	)
 	if err != nil {
-		log.Error(fmt.Sprintf("NewGiteaDownloader: %s", err.Error()))
+		log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
 		return nil, err
 	}
 
@@ -101,12 +102,13 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
 	// set small maxPerPage since we can only guess
 	// (default would be 50 but this can differ)
 	maxPerPage := 10
-	// new gitea instances can tell us what maximum they have
-	if giteaClient.CheckServerVersionConstraint(">=1.13.0") == nil {
-		apiConf, _, err := giteaClient.GetGlobalAPISettings()
-		if err != nil {
-			return nil, err
-		}
+	// gitea instances >=1.13 can tell us what maximum they have
+	apiConf, _, err := giteaClient.GetGlobalAPISettings()
+	if err != nil {
+		log.Info("Unable to get global API settings. Ignoring these.")
+		log.Debug("giteaClient.GetGlobalAPISettings. Error: %v", err)
+	}
+	if apiConf != nil {
 		maxPerPage = apiConf.MaxResponseItems
 	}
 
@@ -324,45 +326,44 @@ func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, er
 }
 
 func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
-	var reactions []*base.Reaction
 	if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
 		log.Info("GiteaDownloader: instance to old, skip getIssueReactions")
-		return reactions, nil
+		return []*base.Reaction{}, nil
 	}
 	rl, _, err := g.client.GetIssueReactions(g.repoOwner, g.repoName, index)
 	if err != nil {
 		return nil, err
 	}
 
-	for _, reaction := range rl {
-		reactions = append(reactions, &base.Reaction{
-			UserID:   reaction.User.ID,
-			UserName: reaction.User.UserName,
-			Content:  reaction.Reaction,
-		})
-	}
-	return reactions, nil
+	return g.convertReactions(rl), nil
 }
 
 func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction, error) {
-	var reactions []*base.Reaction
 	if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
 		log.Info("GiteaDownloader: instance to old, skip getCommentReactions")
-		return reactions, nil
+		return []*base.Reaction{}, nil
 	}
 	rl, _, err := g.client.GetIssueCommentReactions(g.repoOwner, g.repoName, commentID)
 	if err != nil {
 		return nil, err
 	}
 
+	return g.convertReactions(rl), nil
+}
+
+func (g *GiteaDownloader) convertReactions(rl []*gitea_sdk.Reaction) []*base.Reaction {
+	var reactions []*base.Reaction
 	for i := range rl {
+		if rl[i].User.ID <= 0 {
+			continue
+		}
 		reactions = append(reactions, &base.Reaction{
 			UserID:   rl[i].User.ID,
 			UserName: rl[i].User.UserName,
 			Content:  rl[i].Reaction,
 		})
 	}
-	return reactions, nil
+	return reactions
 }
 
 // GetIssues returns issues according start and limit
@@ -394,7 +395,11 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
 
 		reactions, err := g.getIssueReactions(issue.Index)
 		if err != nil {
-			return nil, false, fmt.Errorf("error while loading reactions: %v", err)
+			log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)
+			if err2 := models.CreateRepositoryNotice(
+				fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil {
+				log.Error("create repository notice failed: ", err2)
+			}
 		}
 
 		var assignees []string
@@ -445,13 +450,17 @@ func (g *GiteaDownloader) GetComments(index int64) ([]*base.Comment, error) {
 		// Page:     i,
 	}})
 	if err != nil {
-		return nil, fmt.Errorf("error while listing comments: %v", err)
+		return nil, fmt.Errorf("error while listing comments for issue #%d. Error: %v", index, err)
 	}
 
 	for _, comment := range comments {
 		reactions, err := g.getCommentReactions(comment.ID)
 		if err != nil {
-			return nil, fmt.Errorf("error while listing comment creactions: %v", err)
+			log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", index, comment.ID, g.repoOwner, g.repoName, err)
+			if err2 := models.CreateRepositoryNotice(
+				fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", index, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
+				log.Error("create repository notice failed: ", err2)
+			}
 		}
 
 		allComments = append(allComments, &base.Comment{
@@ -489,7 +498,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 		State: gitea_sdk.StateAll,
 	})
 	if err != nil {
-		return nil, false, fmt.Errorf("error while listing repos: %v", err)
+		return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %v", page, perPage, err)
 	}
 	for _, pr := range prs {
 		var milestone string
@@ -520,7 +529,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 			if headSHA == "" {
 				headCommit, _, err := g.client.GetSingleCommit(g.repoOwner, g.repoName, url.PathEscape(pr.Head.Ref))
 				if err != nil {
-					return nil, false, fmt.Errorf("error while resolving git ref: %v", err)
+					return nil, false, fmt.Errorf("error while resolving head git ref: %s for pull #%d. Error: %v", pr.Head.Ref, pr.Index, err)
 				}
 				headSHA = headCommit.SHA
 			}
@@ -533,7 +542,11 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 
 		reactions, err := g.getIssueReactions(pr.Index)
 		if err != nil {
-			return nil, false, fmt.Errorf("error while loading reactions: %v", err)
+			log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)
+			if err2 := models.CreateRepositoryNotice(
+				fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil {
+				log.Error("create repository notice failed: ", err2)
+			}
 		}
 
 		var assignees []string