Merge pull request libgit2#362 from libgit2/cmn/master-tip-static

Update master to latest libgit2 and build statically

Author: carlosmn

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/libgit2"]
+	path = vendor/libgit2
+	url = https://github.com/libgit2/libgit2

.travis.yml

@@ -5,10 +5,6 @@ sudo: required
 install: ./script/install-libgit2.sh
 
 go:
-  - 1.1
-  - 1.2
-  - 1.3
-  - 1.4
   - 1.5
   - 1.6
   - 1.7

Makefile

@@ -1,8 +1,11 @@
 default: test
 
-test:
+build-libgit2:
+	./script/build-libgit2-static.sh
+
+test: build-libgit2
 	go run script/check-MakeGitError-thread-lock.go
 	go test ./...
 
-install:
+install: build-libgit2
 	go install ./...

README.md

@@ -2,57 +2,59 @@ git2go
 ======
 [![GoDoc](https://godoc.org/github.com/libgit2/git2go?status.svg)](http://godoc.org/github.com/libgit2/git2go) [![Build Status](https://travis-ci.org/libgit2/git2go.svg?branch=master)](https://travis-ci.org/libgit2/git2go)
 
+Go bindings for [libgit2](http://libgit2.github.com/).
 
-Go bindings for [libgit2](http://libgit2.github.com/). The `master` branch follows the latest libgit2 release. The versioned branches indicate which libgit2 version they work against.
+### Which branch to use
 
-Installing
-----------
+The numbered branches work against the version of libgit2 as specified by their number. You can import them in your project via gopkg.in, e.g. if you have libgit2 v0.25 installed you'd import with
 
-This project wraps the functionality provided by libgit2. If you're using a stable version, install it to your system via your system's package manager and then install git2go as usual.
+    import "gopkg.in/libgit2/git2go.v25"
 
-Otherwise (`next` which tracks an unstable version), we need to build libgit2 as well. In order to build it, you need `cmake`, `pkg-config` and a C compiler. You will also need the development packages for OpenSSL and LibSSH2 installed if you want libgit2 to support HTTPS and SSH respectively.
+which will ensure there are no sudden changes to the API.
 
-### Stable version
+The `master` branch follows the tip of libgit2 itself (with some lag) and as such has no guarantees on its own API nor does it have expectations the stability of libgit2's. Thus this only supports statically linking against libgit2.
 
-git2go has `master` which tracks the latest release of libgit2, and versioned branches which indicate which version of libgit2 they work against. Install the development package on your system via your favourite package manager or from source and you can use a service like gopkg.in to use the appropriate version. For the libgit2 v0.22 case, you can use
+Installing
+----------
 
-    import "gopkg.in/libgit2/git2go.v22"
+This project wraps the functionality provided by libgit2. It thus needs it in order to perform the work.
 
-to use a version of git2go which will work against libgit2 v0.22 and dynamically link to the library. You can use
+This project wraps the functionality provided by libgit2. If you're using a versioned branch, install it to your system via your system's package manager and then install git2go.
 
-    import "github.com/libgit2/git2go"
 
-to use the version which works against the latest release.
+### Versioned branch, dynamic linking
 
-### From `next`
+When linking dynamically against a released version of libgit2, install it via your system's package manager. CGo will take care of finding its pkg-config file and set up the linking. Import via gopkg.in, e.g. to work against libgit2 v0.25
 
-The `next` branch follows libgit2's master branch, which means there is no stable API or ABI to link against. git2go can statically link against a vendored version of libgit2.
+    import "gopkg.in/libgit2/git2go.v25"
+
+### Master branch, or static linking
+
+If using `master` or building a branch statically, we need to build libgit2 first. In order to build it, you need `cmake`, `pkg-config` and a C compiler. You will also need the development packages for OpenSSL (outside of Windows or macOS) and LibSSH2 installed if you want libgit2 to support HTTPS and SSH respectively. Note that even if libgit2 is included in the resulting binary, its dependencies will not be.
 
 Run `go get -d github.com/libgit2/git2go` to download the code and go to your `$GOPATH/src/github.com/libgit2/git2go` directory. From there, we need to build the C code and put it into the resulting go binary.
 
-    git checkout next
     git submodule update --init # get libgit2
     make install
 
-will compile libgit2. Run `go install` so that it's statically linked to the git2go package.
+will compile libgit2, link it into git2go and install it.
 
 Parallelism and network operations
 ----------------------------------
 
-libgit2 uses OpenSSL and LibSSH2 for performing encrypted network connections. For now, git2go asks libgit2 to set locking for OpenSSL. This makes HTTPS connections thread-safe, but it is fragile and will likely stop doing it soon. This may also make SSH connections thread-safe if your copy of libssh2 is linked against OpenSSL. Check libgit2's `THREADSAFE.md` for more information.
+libgit2 may use OpenSSL and LibSSH2 for performing encrypted network connections. For now, git2go asks libgit2 to set locking for OpenSSL. This makes HTTPS connections thread-safe, but it is fragile and will likely stop doing it soon. This may also make SSH connections thread-safe if your copy of libssh2 is linked against OpenSSL. Check libgit2's `THREADSAFE.md` for more information.
 
 Running the tests
 -----------------
 
-For the stable version, `go test` will work as usual. For the `next` branch, similarly to installing, running the tests requires linking against the local libgit2 library, so the Makefile provides a wrapper
+For the stable version, `go test` will work as usual. For the `next` branch, similarly to installing, running the tests requires building a local libgit2 library, so the Makefile provides a wrapper that makes sure it's built
 
     make test
 
-Alternatively, if you want to pass arguments to `go test`, you can use the script that sets it all up
-
-    ./script/with-static.sh go test -v
+Alternatively, you can build the library manually first and then run the tests
 
-which will run the specified arguments with the correct environment variables.
+    ./script/build-libgit2-static.sh
+    go test -v
 
 License
 -------

blob.go

@@ -4,15 +4,13 @@ package git
 #include <git2.h>
 #include <string.h>
 
-extern int _go_git_blob_create_fromchunks(git_oid *id,
-	git_repository *repo,
-	const char *hintpath,
-	void *payload);
-
+int _go_git_writestream_write(git_writestream *stream, const char *buffer, size_t len);
+void _go_git_writestream_free(git_writestream *stream);
 */
 import "C"
 import (
 	"io"
+	"reflect"
 	"runtime"
 	"unsafe"
 )
@@ -87,27 +85,71 @@ func blobChunkCb(buffer *C.char, maxLen C.size_t, handle unsafe.Pointer) int {
 	return len(goBuf)
 }
 
-func (repo *Repository) CreateBlobFromChunks(hintPath string, callback BlobChunkCallback) (*Oid, error) {
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
-
+func (repo *Repository) CreateFromStream(hintPath string) (*BlobWriteStream, error) {
 	var chintPath *C.char = nil
+	var stream *C.git_writestream
+
 	if len(hintPath) > 0 {
 		chintPath = C.CString(hintPath)
 		defer C.free(unsafe.Pointer(chintPath))
 	}
-	oid := C.git_oid{}
 
-	payload := &BlobCallbackData{Callback: callback}
-	handle := pointerHandles.Track(payload)
-	defer pointerHandles.Untrack(handle)
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ecode := C.git_blob_create_fromstream(&stream, repo.ptr, chintPath)
+	if ecode < 0 {
+		return nil, MakeGitError(ecode)
+	}
+
+	return newBlobWriteStreamFromC(stream), nil
+}
+
+type BlobWriteStream struct {
+	ptr *C.git_writestream
+}
+
+func newBlobWriteStreamFromC(ptr *C.git_writestream) *BlobWriteStream {
+	stream := &BlobWriteStream{
+		ptr: ptr,
+	}
+
+	runtime.SetFinalizer(stream, (*BlobWriteStream).Free)
+	return stream
+}
+
+// Implement io.Writer
+func (stream *BlobWriteStream) Write(p []byte) (int, error) {
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&p))
+	ptr := (*C.char)(unsafe.Pointer(header.Data))
+	size := C.size_t(header.Len)
 
-	ecode := C._go_git_blob_create_fromchunks(&oid, repo.ptr, chintPath, handle)
-	if payload.Error != nil {
-		return nil, payload.Error
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ecode := C._go_git_writestream_write(stream.ptr, ptr, size)
+	if ecode < 0 {
+		return 0, MakeGitError(ecode)
 	}
+
+	return len(p), nil
+}
+
+func (stream *BlobWriteStream) Free() {
+	runtime.SetFinalizer(stream, nil)
+	C._go_git_writestream_free(stream.ptr)
+}
+
+func (stream *BlobWriteStream) Commit() (*Oid, error) {
+	oid := C.git_oid{}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ecode := C.git_blob_create_fromstream_commit(&oid, stream.ptr)
 	if ecode < 0 {
 		return nil, MakeGitError(ecode)
 	}
+
 	return newOidFromC(&oid), nil
 }

commit.go

@@ -22,6 +22,10 @@ func (c Commit) Message() string {
 	return C.GoString(C.git_commit_message(c.cast_ptr))
 }
 
+func (c Commit) RawMessage() string {
+	return C.GoString(C.git_commit_message_raw(c.cast_ptr))
+}
+
 func (c Commit) Summary() string {
 	return C.GoString(C.git_commit_summary(c.cast_ptr))
 }

describe_test.go

@@ -93,14 +93,16 @@ func TestDescribeCommit(t *testing.T) {
 func checkDescribeNoRefsFound(t *testing.T, err error) {
 	// The failure happens at wherever we were called, not here
 	_, file, line, ok := runtime.Caller(1)
+	expectedString := "no reference found, cannot describe anything"
 	if !ok {
 		t.Fatalf("Unable to get caller")
 	}
-	if err == nil || !strings.Contains(err.Error(), "No reference found, cannot describe anything") {
+	if err == nil || !strings.Contains(err.Error(), expectedString) {
 		t.Fatalf(
-			"%s:%v: was expecting error 'No reference found, cannot describe anything', got %v",
+			"%s:%v: was expecting error %v, got %v",
 			path.Base(file),
 			line,
+			expectedString,
 			err,
 		)
 	}

diff.go

@@ -20,6 +20,7 @@ const (
 	DiffFlagBinary    DiffFlag = C.GIT_DIFF_FLAG_BINARY
 	DiffFlagNotBinary DiffFlag = C.GIT_DIFF_FLAG_NOT_BINARY
 	DiffFlagValidOid  DiffFlag = C.GIT_DIFF_FLAG_VALID_ID
+	DiffFlagExists    DiffFlag = C.GIT_DIFF_FLAG_EXISTS
 )
 
 type Delta int
@@ -34,6 +35,8 @@ const (
 	DeltaIgnored    Delta = C.GIT_DELTA_IGNORED
 	DeltaUntracked  Delta = C.GIT_DELTA_UNTRACKED
 	DeltaTypeChange Delta = C.GIT_DELTA_TYPECHANGE
+	DeltaUnreadable Delta = C.GIT_DELTA_UNREADABLE
+	DeltaConflicted Delta = C.GIT_DELTA_CONFLICTED
 )
 
 type DiffLineType int
@@ -399,6 +402,7 @@ const (
 	DiffIgnoreFilemode         DiffOptionsFlag = C.GIT_DIFF_IGNORE_FILEMODE
 	DiffIgnoreSubmodules       DiffOptionsFlag = C.GIT_DIFF_IGNORE_SUBMODULES
 	DiffIgnoreCase             DiffOptionsFlag = C.GIT_DIFF_IGNORE_CASE
+	DiffIncludeCaseChange      DiffOptionsFlag = C.GIT_DIFF_INCLUDE_CASECHANGE
 
 	DiffDisablePathspecMatch    DiffOptionsFlag = C.GIT_DIFF_DISABLE_PATHSPEC_MATCH
 	DiffSkipBinaryCheck         DiffOptionsFlag = C.GIT_DIFF_SKIP_BINARY_CHECK

features.go

@@ -0,0 +1,30 @@
+package git
+
+/*
+#include <git2.h>
+*/
+import "C"
+
+type Feature int
+
+const (
+	// libgit2 was built with threading support
+	FeatureThreads Feature = C.GIT_FEATURE_THREADS
+
+	// libgit2 was built with HTTPS support built-in
+	FeatureHttps Feature = C.GIT_FEATURE_HTTPS
+
+	// libgit2 was build with SSH support built-in
+	FeatureSsh Feature = C.GIT_FEATURE_SSH
+
+	// libgit2 was built with nanosecond support for files
+	FeatureNSec Feature = C.GIT_FEATURE_NSEC
+)
+
+// Features returns a bit-flag of Feature values indicating which features the
+// loaded libgit2 library has.
+func Features() Feature {
+	features := C.git_libgit2_features()
+
+	return Feature(features)
+}

git.go

@@ -1,12 +1,15 @@
 package git
 
 /*
+#cgo CFLAGS: -I${SRCDIR}/vendor/libgit2/include
+#cgo LDFLAGS: -L${SRCDIR}/vendor/libgit2/build/ -lgit2
+#cgo windows LDFLAGS: -lwinhttp
+#cgo !windows pkg-config: --static ${SRCDIR}/vendor/libgit2/build/libgit2.pc
 #include <git2.h>
 #include <git2/sys/openssl.h>
-#cgo pkg-config: libgit2
 
-#if LIBGIT2_VER_MAJOR != 0 || LIBGIT2_VER_MINOR != 24
-# error "Invalid libgit2 version; this git2go supports libgit2 v0.24"
+#if LIBGIT2_VER_MAJOR != 0 || LIBGIT2_VER_MINOR != 25
+# error "Invalid libgit2 version; this git2go supports libgit2 v0.25"
 #endif
 
 */
@@ -125,6 +128,15 @@ func init() {
 
 	C.git_libgit2_init()
 
+	// Due to the multithreaded nature of Go and its interaction with
+	// calling C functions, we cannot work with a library that was not built
+	// with multi-threading support. The most likely outcome is a segfault
+	// or panic at an incomprehensible time, so let's make it easy by
+	// panicking right here.
+	if Features()&FeatureThreads == 0 {
+		panic("libgit2 was not built with threading support")
+	}
+
 	// This is not something we should be doing, as we may be
 	// stomping all over someone else's setup. The user should do
 	// this themselves or use some binding/wrapper which does it

git_test.go

@@ -93,6 +93,8 @@ func updateReadme(t *testing.T, repo *Repository, content string) (*Oid, *Oid) {
 	checkFatal(t, err)
 	err = idx.AddByPath("README")
 	checkFatal(t, err)
+	err = idx.Write()
+	checkFatal(t, err)
 	treeId, err := idx.WriteTree()
 	checkFatal(t, err)
 

odb.go

@@ -226,6 +226,10 @@ func (object *OdbObject) Len() (len uint64) {
 	return uint64(C.git_odb_object_size(object.ptr))
 }
 
+func (object *OdbObject) Type() ObjectType {
+	return ObjectType(C.git_odb_object_type(object.ptr))
+}
+
 func (object *OdbObject) Data() (data []byte) {
 	var c_blob unsafe.Pointer = C.git_odb_object_data(object.ptr)
 	var blob []byte