Commit 3050a4eb5722a09b6bdd8919e698815f1be4feb9 - kitchensink-clojure

+2

-5

.travis.yml less more

0	0	language: clojure
1		lein: 2.7.1
2	1	jdk:
3		- oraclejdk7
4		- openjdk7
5		- openjdk6
6		script: ./ext/travisci/test.sh
	2	- openjdk8
	3	- openjdk11
7	4	notifications:
8	5	email: false
9	6	hipchat:

+47

-1

CHANGELOG.md less more

	0	## 3.1.0
	1
	2	This is a minor feature release.
	3
	4	* Add a function for atomic file writes
	5
	6	## 3.0.0
	7
	8	Maintenance:
	9	* Update dependencies to take up clj-parent 4
	10
	11	## 2.5.2
	12
	13	This is a minor maintenance release.
	14
	15	Maintenance:
	16	* Fix adding URLs to classpath under Java 9.
	17
	18	## 2.5.1
	19
	20	This is a minor maintenance release.
	21
	22	Maintenance:
	23	* Fix symbol redef warnings under Clojure 1.9
	24
	25	## 2.5.0
	26
	27	This is a minor feature release.
	28
	29	Features:
	30	* add a `stream->sha256` function for hashing the contents of an InputStream
	31
	32	## 2.4.0
	33
	34	This is a minor feature and improvement release.
	35
	36	Features:
	37	* add a `utf8-string->sha256` function, directly analogous to `utf8-string->sha1`
	38	* add a `file->sha256` function, equivalent to reading a file's contents as a
	39	UTF-8 string and hashing the result. Uses an InputStream internally to avoid
	40	reading the entire file into memory at once.
	41
	42	Improvement:
	43	* the `open-port-num` function should now return a random port number from the
	44	entire traditional ephemeral port range of 49152 through 65535.
	45
0	46	## 2.3.0
1	47
2	48	This is a minor feature release.

20	66
21	67	Maintenance:
22	68
23		* Update to dynapath 0.2.5, to address some compatability issues with Java 9.
	69	* Update to dynapath 0.2.5, to address some compatibility issues with Java 9.
24	70
25	71	## 2.1.1
26	72

+4

-0

CODEOWNERS less more

	0	# This will cause the puppetserver-maintainers group to be assigned
	1	# review of any opened PRs against the branches containing this file.
	2
	3	* @puppetlabs/puppetserver-maintainers

+0

-23

~~MAINTAINERS~~ less more

0		{
1		"version": 1,
2		"file_format": "This MAINTAINERS file format is described at http://pup.pt/maintainers",
3		"issues": "https://tickets.puppetlabs.com/browse/TK",
4		"internal_list": "https://groups.google.com/a/puppet.com/forum/?hl=en#!forum/discuss-trapperkeeper-maintainers",
5		"people": [
6		{
7		"github": "senior",
8		"email": "ryan.senior@puppet.com",
9		"name": "Ryan Senior"
10		},
11		{
12		"github": "pcarlisle",
13		"email": "pcarlisle@puppet.com",
14		"name": "Patrick Carlisle"
15		},
16		{
17		"github": "camlow325",
18		"email": "jeremy.barlow@puppet.com",
19		"name": "Jeremy Barlow"
20		}
21		]
22		}

+0

-0

classpath-test/does-not-exist-anywhere-else less more

(New empty file)

+0

-3

~~ext/travisci/test.sh~~ less more

0		#!/bin/bash
1
2		lein2 test

+9

-7

project.clj less more

0		(defproject puppetlabs/kitchensink "2.3.0"
	0	(defproject puppetlabs/kitchensink "3.1.0"
1	1	:description "Clojure utility functions"
2	2	:license {:name "Apache License, Version 2.0"
3	3	:url "http://www.apache.org/licenses/LICENSE-2.0.html"}
4	4
5		:min-lein-version "2.7.1"
	5	:min-lein-version "2.9.1"
6	6
7		:parent-project {:coords [puppetlabs/clj-parent "0.1.3"]
	7	:parent-project {:coords [puppetlabs/clj-parent "4.0.0"]
8	8	:inherit [:managed-dependencies]}
9	9
10	10	;; Abort when version ranges or version conflicts are detected in

17	17	[org.clojure/tools.cli]
18	18
19	19	[clj-time]
20		[me.raynes/fs]
	20	[clj-commons/fs]
21	21	[slingshot]
22	22	[cheshire]
23	23
24	24	[org.ini4j/ini4j "0.5.2"]
25		[org.tcrawley/dynapath "0.2.5"]
	25	[org.tcrawley/dynapath]
26	26	[digest "1.4.3"]
27
28	27	]
29	28
30	29	;; By declaring a classifier here and a corresponding profile below we'll get an additional jar

37	36
38	37	;; this plugin is used by jenkins jobs to interrogate the project version
39	38	:plugins [[lein-project-version "0.1.0"]
40		[lein-parent "0.3.1"]]
	39	[lein-parent "0.3.7"]]
	40
	41	:test-selectors {:default (complement :slow)
	42	:slow :slow}
41	43
42	44	:deploy-repositories [["releases" {:url "https://clojars.org/repo"
43	45	:username :env/clojars_jenkins_username

+10

-0

src/puppetlabs/kitchensink/classpath.clj less more

34	34	[cl]
35	35	(dp/addable-classpath? cl))
36	36
	37	(defn- ensure-modifiable-classloader
	38	"Check if there is a modifiable classloader in the current hierarchy, and add
	39	one if not."
	40	[]
	41	(let [classloader (.. Thread currentThread getContextClassLoader)]
	42	(when (not-any? modifiable-classloader? (classloader-hierarchy classloader))
	43	(let [new-cl (clojure.lang.DynamicClassLoader. classloader)]
	44	(.. Thread currentThread (setContextClassLoader new-cl))))))
	45
37	46	(defn add-classpath
38	47	"A corollary to the (deprecated) `add-classpath` in clojure.core. This implementation
39	48	requires a java.io.File or String path to a jar file or directory, and will attempt

56	65	(if-not (dp/add-classpath-url classloader (.toURL (file jar-or-dir)))
57	66	(throw (IllegalStateException. (str classloader " is not a modifiable classloader")))))
58	67	([jar-or-dir]
	68	(ensure-modifiable-classloader)
59	69	(let [classloaders (classloader-hierarchy)]
60	70	(if-let [cl (last (filter modifiable-classloader? classloaders))]
61	71	(add-classpath jar-or-dir cl)

+75

-39

src/puppetlabs/kitchensink/core.clj less more

4	4	;; altogether. But who has time for that?
5	5
6	6	(ns puppetlabs.kitchensink.core
7		(:import [org.ini4j Ini Config BasicProfileSection]
8		[javax.naming.ldap LdapName]
9		[java.io StringWriter Reader File])
10		(:require [clojure.tools.logging :as log]
	7	(:refer-clojure :exclude [boolean? uuid?])
	8	(:require [clj-time.coerce :as time-coerce]
	9	[clj-time.core :as time]
	10	[clj-time.format :as time-format]
	11	[clojure.java.io :as io]
	12	[clojure.pprint :as pprint]
	13	[clojure.set :as set]
11	14	[clojure.string :as string]
12	15	[clojure.tools.cli :as cli]
13		[clojure.java.io :as io]
14		[digest]
15		[slingshot.slingshot :refer [throw+]]
16		[me.raynes.fs :as fs])
17		(:use [clojure.java.io :only (reader)]
18		[clojure.set :only (difference union)]
19		[clojure.string :only (split)]
20		[clojure.stacktrace :only (print-cause-trace)]
21		[clojure.pprint :only [pprint]]
22		[clj-time.core :only [now seconds minutes hours days years]]
23		[clj-time.coerce :only [ICoerce to-date-time]]
24		[clj-time.format :only [formatters unparse]]))
25
	16	[clojure.tools.logging :as log]
	17	digest
	18	[me.raynes.fs :as fs]
	19	[slingshot.slingshot :refer [throw+]])
	20	(:import [java.io File Reader StringWriter]
	21	javax.naming.ldap.LdapName
	22	[org.ini4j BasicProfileSection Config Ini]))
26	23
27	24	(defn error-map
28	25	[kind message]

43	40	convertible to a Joda DateTime"
44	41	[x]
45	42	(and
46		(satisfies? ICoerce x)
47		(to-date-time x)))
	43	(satisfies? time-coerce/ICoerce x)
	44	(time-coerce/to-date-time x)))
48	45
49	46	(defn boolean?
50	47	"Returns true if the value is a boolean"

114	111
115	112	(defn pprint-to-string [x]
116	113	(let [w (StringWriter.)]
117		(pprint x w)
	114	(pprint/pprint x w)
118	115	(.toString w)))
119	116
120	117	(defn to-sentence

149	146	(defn lines
150	147	"Returns a sequence of lines from the given filename"
151	148	[filename]
152		(with-open [file-reader (reader (fs/file filename))]
	149	(with-open [file-reader (io/reader (fs/file filename))]
153	150	;; line seq is lazy and file-reader gets closed
154	151	(doall (line-seq file-reader))))
155	152

302	299	;; ## Collection operations
303	300
304	301	(defn symmetric-difference
305		"Computes the symmetric difference between 2 sets"
	302	"Computes the symmetric set/difference between 2 sets"
306	303	[s1 s2]
307		(union (difference s1 s2) (difference s2 s1)))
	304	(set/union (set/difference s1 s2) (set/difference s2 s1)))
308	305
309	306	(defn as-collection
310	307	"Returns the item wrapped in a collection, if it's not one

602	599	"Returns a timestamp string for the given `time`, or the current time if none
603	600	is provided. The format of the timestamp is eg. 2012-02-23T22:01:39.539Z."
604	601	([]
605		(timestamp (now)))
	602	(timestamp (time/now)))
606	603	([time]
607		(unparse (formatters :date-time) time)))
	604	(time-format/unparse (time-format/formatters :date-time) time)))
608	605
609	606	;; ## Exception handling
610	607

739	736	{} section))
740	737
741	738	(defn parse-ini
742		"Takes a reader that contains an ini file, and returns an Ini object
	739	"Takes a io/reader that contains an ini file, and returns an Ini object
743	740	containing the parsed results"
744	741	[^Reader ini-reader]
745	742	{:pre [(instance? Reader ini-reader)]

763	760	(every? keyword? (keys %))
764	761	(every? map? (vals %))]}
765	762
766		(with-open [ini (reader filename)]
	763	(with-open [ini (io/reader filename)]
767	764	(reduce (fn [acc [name section]]
768	765	(assoc acc
769	766	(keywordize name)

964	961	(let [bytes (.getBytes s "UTF-8")]
965	962	(digest/sha-1 [bytes])))
966	963
	964	(defn utf8-string->sha256
	965	"Compute a SHA-256 hash for the UTF-8 encoded version of the supplied
	966	string"
	967	[^String s]
	968	{:pre [(string? s)]
	969	:post [(string? %)]}
	970	(let [bytes (.getBytes s "UTF-8")]
	971	(digest/sha-256 [bytes])))
	972
	973	(defn stream->sha256
	974	"Compute a SHA-256 hash for the given java.io.InputStream object."
	975	[^java.io.InputStream stream]
	976	{:pre [(instance? java.io.InputStream stream)]
	977	:post [(string? %)]}
	978	(digest/sha-256 stream))
	979
	980	(defn file->sha256
	981	"Compute a SHA-256 hash for the given java.io.File object.
	982	Uses an InputStream to read the file, so it doesn't load all the contents into
	983	memory at once."
	984	[^java.io.File file]
	985	{:pre [(instance? java.io.File file)]
	986	:post [(string? %)]}
	987	(digest/sha-256 file))
	988
967	989	(defn bounded-memoize
968	990	"Similar to memoize, but the cache will be reset if the number of entries
969	991	exceeds the specified `bound`."

1017	1039	(string? b)]
1018	1040	:post [(number? %)]}
1019	1041	(let [parse #(mapv parse-int (-> %
1020		(split #"-")
	1042	(string/split #"-")
1021	1043	(first)
1022		(split #"[\\._]")))]
	1044	(string/split #"[\\._]")))]
1023	1045	(compare (parse a) (parse b))))
1024	1046
1025	1047	(def java-version

1066	1088	~@(interleave (repeat g) (map pstep forms))]
1067	1089	~g)))
1068	1090
	1091	(defn- port-open?
	1092	[port]
	1093	(try
	1094	(with-open [_ (java.net.ServerSocket. port)]
	1095	port)
	1096	(catch java.net.BindException e
	1097	(when-not (re-find #"already in use" (.getMessage e))
	1098	(throw e)))))
	1099
1069	1100	(defn open-port-num
1070		"Returns a currently open port number"
	1101	"Returns a currently open port number in the traditional ephemeral port range
	1102	of 49152 through 65535."
1071	1103	[]
1072		(with-open [s (java.net.ServerSocket. 0)]
1073		(.getLocalPort s)))
	1104	(let [lo 49152
	1105	hi 65536] ; one higher because the upper limit is exclusive
	1106	(if-let [open-port (some port-open? (shuffle (range lo hi)))]
	1107	open-port
	1108	(throw (java.net.BindException.
	1109	"All ephemeral ports are already in use (Bind failed)")))))
1074	1110
1075	1111	(defmacro assoc-if-new
1076	1112	"Assocs the provided values with the corresponding keys if and only

1098	1134	(defn parse-interval
1099	1135	"Given a time string of the form \"<number>\", or \"<number><unit>\", this
1100	1136	function parses this time amount and returns a joda time Period instance.
1101		If the unit is left off, the units are assumed to be seconds
	1137	If the unit is left off, the units are assumed to be time/seconds
1102	1138
1103	1139	Example: \"12h\" -> (clj-time.core/hours 12)
1104	1140	Example: \"12\" -> (clj-time.core/seconds 12)

1111	1147	(when-let [[_ num unit] (re-matches #"^(\d+)([smhdy]?)$" time-str)]
1112	1148	(let [num (parse-int num)
1113	1149	time-fn (case unit
1114		"s" seconds
1115		"m" minutes
1116		"h" hours
1117		"d" days
1118		"y" years
1119		"" seconds)]
	1150	"s" time/seconds
	1151	"m" time/minutes
	1152	"h" time/hours
	1153	"d" time/days
	1154	"y" time/years
	1155	"" time/seconds)]
1120	1156	(time-fn num)))))

+84

-0

src/puppetlabs/kitchensink/file.clj less more

	0	(ns puppetlabs.kitchensink.file
	1	(:import [java.io BufferedWriter FileOutputStream OutputStreamWriter]
	2	[java.nio.file CopyOption Files LinkOption Paths StandardCopyOption]
	3	[java.nio.file.attribute FileAttribute PosixFilePermissions]))
	4
	5	(defn str->path
	6	[path]
	7	(Paths/get path (into-array String [])))
	8
	9	(defn get-perms
	10	"Returns the currently set permissions of the given file path."
	11	[path]
	12	(-> (str->path path)
	13	(Files/getPosixFilePermissions (into-array LinkOption []))
	14	PosixFilePermissions/toString))
	15
	16	(defn set-perms
	17	"Set the provided permissions on the given path. The permissions string is in
	18	the form of the standard 9 character posix format, e.g. \"rwxr-xr-x\"."
	19	[path permissions]
	20	(-> (str->path path)
	21	(Files/setPosixFilePermissions
	22	(PosixFilePermissions/fromString permissions))
	23	(.toFile)))
	24
	25	(defn perms->attribute
	26	[permissions]
	27	(PosixFilePermissions/asFileAttribute (PosixFilePermissions/fromString permissions)))
	28
	29	(def nofollow-links
	30	(into-array LinkOption [LinkOption/NOFOLLOW_LINKS]))
	31
	32	(def ^:private default-permissions
	33	(PosixFilePermissions/fromString "rw-r-----"))
	34
	35	(defn atomic-write
	36	"Write to a file atomically. This takes a function that should operate on a
	37	Writer as its first argument. If permissions are specified (as a string of the
	38	form \"rwxrwxrwx\") then the file will be created with those permissions. If
	39	not specified and the file already exists then existing permissions will be
	40	preserved. If no file exist a default is set."
	41	([path write-function]
	42	(atomic-write path write-function nil))
	43	([path write-function permissions]
	44	(let [target (str->path path)
	45	dir (.getParent target)
	46	file-exists? (Files/exists target nofollow-links)
	47	owner (if file-exists?
	48	(Files/getOwner target nofollow-links))
	49	group (if file-exists?
	50	(Files/getAttribute target "posix:group" nofollow-links))
	51	permissions (if permissions
	52	(PosixFilePermissions/fromString permissions)
	53	(if file-exists?
	54	(Files/getPosixFilePermissions target nofollow-links)
	55	default-permissions))
	56	temp-attributes (into-array FileAttribute [(perms->attribute "rw-------")])
	57	temp-file (Files/createTempFile dir (.toString (.getFileName target)) "tmp" temp-attributes)
	58	stream (proxy [FileOutputStream] [(.toString temp-file)]
	59	(close []
	60	(.sync (.getFD this))
	61	(proxy-super close)))
	62	writer (BufferedWriter. (OutputStreamWriter. stream))]
	63
	64	(write-function writer)
	65
	66	(.close writer)
	67
	68	(when owner
	69	(Files/setOwner temp-file owner))
	70	(when group
	71	(Files/setAttribute temp-file "posix:group" group nofollow-links))
	72
	73	;; We set these here instead of at file creation to avoid problems with
	74	;; read-only files and umask
	75	(Files/setPosixFilePermissions temp-file permissions)
	76	(Files/move temp-file target (into-array CopyOption [StandardCopyOption/ATOMIC_MOVE])))))
	77
	78	(defn atomic-write-string
	79	"Write a string to a file atomically. See atomic-write for more details."
	80	([path string]
	81	(atomic-write-string path string nil))
	82	([path string permissions]
	83	(atomic-write path #(.write % string) permissions)))

+9

-11

test/puppetlabs/kitchensink/classpath_test.clj less more

3	3	(:import (java.net URL)))
4	4
5	5	(deftest with-additional-classpath-entries-test
6		(let [paths ["/foo" "/bar"]
7		get-urls #(into #{}
8		(.getURLs (.getContextClassLoader (Thread/currentThread))))]
	6	(let [paths ["classpath-test"]
	7	get-resource #(-> (Thread/currentThread)
	8	.getContextClassLoader
	9	(.getResource "does-not-exist-anywhere-else"))]
9	10	(with-additional-classpath-entries
10	11	paths
11		(testing "classloader now includes the new paths"
12		(let [urls (get-urls)]
13		(is (contains? urls (URL. "file:/foo")))
14		(is (contains? urls (URL. "file:/bar"))))))
15		(testing "classloader has been restored to its previous state"
16		(let [urls (get-urls)]
17		(is (not (contains? urls (URL. "file:/foo"))))
18		(is (not (contains? urls (URL. "file:/bar"))))))))
	12	(testing "classloader now includes the new path"
	13	(is (get-resource))))
	14
	15	(testing "classloader no longer includes the new path"
	16	(is (not (get-resource))))))

+46

-3

test/puppetlabs/kitchensink/core_test.clj less more

6	6	[clj-time.core :as t]
7	7	[puppetlabs.kitchensink.testutils :as testutils]
8	8	[clojure.zip :as zip])
9		(:import (java.util ArrayList)))
	9	(:import (java.util ArrayList)
	10	(java.io ByteArrayInputStream)))
10	11
11	12	(deftest array?-test
12	13	(testing "array?"

415	416
416	417	(testing "should produce the correct hash"
417	418	(is (= "8843d7f92416211de9ebb963ff4ce28125932878"
418		(utf8-string->sha1 "foobar"))))))
	419	(utf8-string->sha1 "foobar")))))
	420
	421	(testing "Computing a SHA-256 for a UTF-8 string"
	422	(testing "should fail if not passed a string"
	423	(is (thrown? AssertionError (utf8-string->sha256 1234))))
	424
	425	(testing "should produce a stable hash"
	426	(is (= (utf8-string->sha256 "foobar")
	427	(utf8-string->sha256 "foobar"))))
	428
	429	(testing "should produce the correct hash"
	430	(is (= "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2"
	431	(utf8-string->sha256 "foobar"))))))
	432
	433	(deftest stream-hashing
	434	(testing "Computing a SHA-256 hash for an input stream"
	435	(testing "should fail if not passed an input stream"
	436	(is (thrown? AssertionError (stream->sha256 "what"))))
	437
	438	(let [stream-fn #(ByteArrayInputStream. (.getBytes "foobar" "UTF-8"))]
	439	(testing "should produce a stable hash"
	440	(is (= (stream->sha256 (stream-fn))
	441	(stream->sha256 (stream-fn)))))
	442
	443	(testing "should produce the correct hash"
	444	(is (= "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2"
	445	(stream->sha256 (stream-fn))))))))
	446
	447	(deftest file-hashing
	448	(testing "Computing a SHA-256 hash for a file"
	449	(testing "should fail if not passed a file"
	450	(is (thrown? AssertionError (file->sha256 "what"))))
	451
	452	(let [f (temp-file "sha256" ".txt")]
	453	(spit f "foobar")
	454
	455	(testing "should produce a stable hash"
	456	(is (= (file->sha256 f)
	457	(file->sha256 f))))
	458
	459	(testing "should produce the correct hash"
	460	(is (= "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2"
	461	(file->sha256 f)))))))
419	462
420	463	(deftest temp-file-name-test
421	464	(testing "The file should not exist."

748	791	(with-timeout 1 false
749	792	(wait-return 1005 true))))))))
750	793
751		(deftest open-port-num-test
	794	(deftest ^:slow open-port-num-test
752	795	(let [port-in-use (open-port-num)]
753	796	(with-open [s (java.net.ServerSocket. port-in-use)]
754	797	(let [open-ports (set (take 60000 (repeatedly open-port-num)))]

+59

-0

test/puppetlabs/kitchensink/file_test.clj less more

	0	(ns puppetlabs.kitchensink.file-test
	1	(:require [clojure.test :refer :all]
	2	[me.raynes.fs :as fs]
	3	[puppetlabs.kitchensink.file :refer :all]))
	4
	5	(deftest atomic-write-test
	6	(testing "when the file doesn't exist"
	7	(let [tmp-file (.toString (fs/temp-file "atomic-writes-test"))
	8	content "Something said, not good"]
	9	(fs/delete tmp-file)
	10	(atomic-write tmp-file #(.write % content))
	11
	12	(testing "it writes the data"
	13	(is (= content (slurp tmp-file))))
	14
	15	(testing "it applies a default mode of 0640"
	16	(is (= "rw-r-----" (get-perms tmp-file))))
	17
	18	(testing "it applies the specified mode"
	19	(fs/delete tmp-file)
	20	(atomic-write tmp-file #(.write % content) "rwxrwxrwx")
	21	(is (= "rwxrwxrwx" (get-perms tmp-file))))
	22
	23	(testing "it can create a read-only file"
	24	(fs/delete tmp-file)
	25	(atomic-write tmp-file #(.write % content) "r--------")
	26	(is (= "r--------" (get-perms tmp-file))))
	27
	28	(fs/delete tmp-file)))
	29
	30	(testing "when overwriting"
	31	(let [tmp-file (.toString (fs/temp-file "atomic-writes-test"))
	32	content "Something said, not good"]
	33	(set-perms tmp-file "rwxr-x--x")
	34
	35	(testing "it preserves existing permissions"
	36	(atomic-write tmp-file #(.write % content))
	37	(is (= "rwxr-x--x" (get-perms tmp-file))))
	38
	39	(testing "it overwrites existing permissions if specified"
	40	(atomic-write tmp-file #(.write % content) "rw-rw----")
	41	(is (= "rw-rw----" (get-perms tmp-file))))
	42
	43	(testing "it updates a read-only file"
	44	(set-perms tmp-file "r--------")
	45	(atomic-write tmp-file #(.write % content))
	46	(is (= content (slurp tmp-file))))
	47
	48	(fs/delete tmp-file)))
	49
	50	(testing "the supplied function is allowed to close the writer"
	51	(let [tmp-file (.toString (fs/temp-file "atomic-writes-test"))
	52	write-fn (fn [writer]
	53	(.write writer "They called me slow")
	54	(.close writer))]
	55
	56	(atomic-write tmp-file write-fn)
	57
	58	(fs/delete tmp-file))))