-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathopensim2git_kde.py
198 lines (160 loc) · 6.92 KB
/
opensim2git_kde.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
import os
from os.path import join
import shutil
import sys
import time
from common import *
# Preliminaries.
# --------------
start_time = time.time()
# This is where we'll put the git repositories.
git_repos_dir = os.path.join(local_dir, 'kde_git_repos')
opensim_svn_url = 'https://simtk.org/svn/opensim'
# We need a local mirror of the svn repository.
svn_mirror_dir = join(local_dir, 'svn_mirror')
# Obtain dependencies. Requires sudo access.
myprint('Obtaining dependencies...')
call('sudo apt-get install svn-all-fast-export dos2unix curl')
# svn mirror.
# -----------
myprint('Setting up svn mirror of original OpenSim repository...')
if not os.path.exists(join(svn_mirror_dir, 'hooks', 'pre-revprop-change')):
# http://cournape.wordpress.com/2007/12/18/making-a-local-mirror-of-a-subversion-repository-using-svnsync/
os.makedirs(svn_mirror_dir)
# Create the repository.
call('svnadmin create %s' % svn_mirror_dir)
# Just to see if the repository is in the correct location.
call('svn info file://%s' % svn_mirror_dir)
# Create a hook so that you can modify revprops, which is necessary for the
# mirror.
# Creating an svn mirror requires a simtk user name.
username = raw_input('Enter your simtk username: ')
# If the mirror hasn't been made yet, make it.
# We'll use this in a bit:
revprop_hook_content = """#!/bin/sh
USER="$3"
if [ "$USER" = "{0}" ]; then exit 0; fi
echo "Only the {0} user can change revprops" >&2
exit 1
""".format(username)
hook = open(join(svn_mirror_dir, 'hooks', 'pre-revprop-change'), 'w')
hook.write(revprop_hook_content)
hook.close()
# Make the hook executable.
call('chmod +x %s/hooks/pre-revprop-change' % svn_mirror_dir)
# Set up the synchronization.
call('svnsync init --username %s file://%s %s' % (username, svn_mirror_dir,
opensim_svn_url))
# Sync the mirror.
call('svnsync sync file://%s' % svn_mirror_dir)
# Create git repositories.
# ------------------------
# To prevent weird overwriting from occurring, remove the previous work.
if os.path.exists(git_repos_dir):
input = raw_input('Deleting pre-existing %s. Okay? (y/n) ' % git_repos_dir)
if input[0] == 'y':
shutil.rmtree(git_repos_dir)
else:
sys.exit('Aborting. Will not overwrite pre-existing work.')
# Make git_repos_dir. Do this before we require the user to `sudo`, so that
# we know the user has permission to write in git_repos_dir.
os.makedirs(git_repos_dir)
# We want the repo to end up in git_repos_dir, so we cd there.
with cd(git_repos_dir):
# Run svn-all-fast-export.
myprint('Creating git repositories...')
# Write output to log files.
out = open('svn-all-fast-export_progress_log.txt', 'w')
err = open('svn-all-fast-export_error_log.txt', 'w')
call("svn-all-fast-export "
"--rules {0}/rules.conf "
"--debug-rules "
"--add-metadata "
"--identity-map {0}/authors.txt "
"--stats "
# "--resume-from 8400 "
# "--max-rev 1095 "
"{1}".format(homebase_dir, svn_mirror_dir),
stdout=out,
stderr=err,
)
# TODO the resulting cfsqp repository does not contain sampl files.
out.close()
err.close()
# Get working copies.
myprint('Create working-copy clones of new git repositories...')
call("git clone cfsqp cfsqp-working-copy")
call("git clone opensim-core opensim-core-working-copy")
# Edit 'description' file, which is used by GitWeb (run `$ git instaweb`).
call('echo "opensim-core: %s" '
'> %s/opensim-core-working-copy/.git/description' % (
opensim_core_description, git_repos_dir))
call('echo "cfsqp: %s" > %s/cfsqp-working-copy/.git/description' % (
cfsqp_description, git_repos_dir))
# Normalize line endings.
# Convert all line endings from CRLF (windows) to LF (unix).
# http://blog.gyoshev.net/2013/08/normalizing-line-endings-in-git-repositories/
# There's an easy way to do this, but it has a significant downside: if you
# use blame on any line after this change, it'll point to this one commit
# that changed the line endings. We use a different method: rewrite the
# history.
myprint('Normalizing line endings...')
fspath = '/dev/shm/opensim2git_temp_repo'
class cd_normalize(cd):
def __enter__(self, *args):
if os.path.exists(fspath): os.removedirs(fspath)
super(cd_normalize, self).__enter__(*args)
def __exit__(self, *args):
if os.path.exists(fspath): os.removedirs(fspath)
super(cd_normalize, self).__exit__(*args)
def filter_branch_tasks(repo_name):
repo_path = join(git_repos_dir, repo_name)
with cd_normalize(repo_path):
# Fix all inconsistent line endings from the history.
# '-d' flag makes in-memory file system: should speed up the operation.
# Also, remove certain files from the history.
# https://help.github.com/articles/remove-sensitive-data
call("git filter-branch "
"--force "
"--tree-filter '%s/normalize_line_endings.sh' "
"--index-filter "
"'git rm -r --cached --ignore unmatch "
"OpenSim/Wrapping/Java/OpenSimJNI/OpenSimJNI_wrap.* "
"OpenSim/Java/OpenSimJNI/OpenSimJNI_wrap.* "
#"Vendors/CFSQP "
"' "
"--prune-empty "
"--tag-name-filter cat "
"-d %s "
"-- --all" % (homebase_dir, fspath),
)
# Ensure consistent line endings in the future.
# http://stackoverflow.com/questions/1510798/trying-to-fix-line-endings-with-git-filter-branch-but-having-no-luck/1511273#1511273
call('echo "* text=auto" >> .gitattributes')
call('git add .gitattributes')
call('git commit -m "Introduce end-of-line normalization."')
filter_branch_tasks('cfsqp-working-copy')
filter_branch_tasks('opensim-core-working-copy')
# Garbage collect.
# ----------------
# Make the repositories smaller.
myprint('Running git garbage collection to reduce repository size...')
def git_garbage_collection(repo_name):
repo_path = join(git_repos_dir, repo_name)
with cd(repo_path):
# Remove backup of the files we deleted from the history.
call('rm -rf .git/refs/original')
call('git reflog expire --expire=now --all')
call('git gc --prune=now')
call('git gc --aggressive --prune=now')
git_garbage_collection('cfsqp-working-copy')
git_garbage_collection('opensim-core-working-copy')
# Make CFSQP a standalone project.
# --------------------------------
with cd(join(git_repos_dir, 'cfsqp-working-copy')):
call('cp %s/CMakeLists_cfsqp_standalone.txt CMakeLists.txt' % homebase_dir)
call('git add CMakeLists.txt')
call('git commit -m"Make this project standalone."')
# Tell the user how long opensim2git ran for.
elapsed_time = time.time() - start_time
myprint("Took %.1f minutes." % (elapsed_time / 60.0))