Révision f5ddb21e
Ajouté par Julien Enselme il y a plus de 10 ans
scripts_divers/migrer_taches_vers_redmine/README | ||
---|---|---|
1 |
* Migrate |
|
2 |
Simply use the following commands in a clean environment: the folder in which |
|
3 |
you launch these commands must not contains any csv or txt files from a previous |
|
4 |
migration otherwise fix-db.rb won't work. |
|
5 |
- python3 post_redmine.py |
|
6 |
- ruby fix-db.rb |
|
7 |
|
|
8 |
* Clean |
|
9 |
Use in a python3 interpreter |
|
10 |
|
|
11 |
#BEGIN_SRC python |
|
12 |
import post_redmine |
|
13 |
redmine = post_redmine.Redmine() |
|
14 |
redmine.sweep() |
|
15 |
#END_SRC |
|
16 |
|
|
17 |
* Export as html |
|
18 |
The export-issues-html.sh script allows you to export issues for a projet as |
|
19 |
html files. It requires access to the database of redmine and only work for |
|
20 |
mysql. |
|
21 |
|
|
22 |
export-issues-html.sh PROJECT_ID |
scripts_divers/migrer_taches_vers_redmine/export-issues-html.sh | ||
---|---|---|
1 |
#!/bin/bash |
|
2 |
|
|
3 |
mysql -h debian -e "SELECT id FROM issues WHERE project_id = $1" -u root -ptata redmine_default > issues_id_list |
|
4 |
|
|
5 |
mkdir issues_html |
|
6 |
cd issues_html |
|
7 |
while read line; do |
|
8 |
wget http://debian/redmine/issues/$line > /dev/null |
|
9 |
done < ../issues_id_list |
scripts_divers/migrer_taches_vers_redmine/fix-db.pl | ||
---|---|---|
1 |
# In order to complete the migration to Redmine, we need to preserve date |
|
2 |
# and username. To do this, we need to fix the database. |
|
3 |
|
|
4 |
use strict; |
|
5 |
use warnings; |
|
6 |
use DBI; |
|
7 |
|
|
8 |
my $db = 'redmine_default'; |
|
9 |
my $server = 'localhost'; # Il est possible de mettre une adresse IP |
|
10 |
my $user = 'root'; # identifiant |
|
11 |
my $passwd = 'tata'; |
|
12 |
my $port = ''; |
|
13 |
|
|
14 |
|
|
15 |
##### Create DB connection |
|
16 |
my $dbh = DBI->connect( "DBI:mysql:database=$db;host=$server;port=$port", $user, $passwd, { RaiseError => 1, } ) or die "Connection impossible à la base de données $db !\n $! \n $@\n$DBI::errstr"; |
|
17 |
|
|
18 |
|
|
19 |
|
|
20 |
###### Initialisation of hashes needed to fix URL |
|
21 |
open(my $fix_url_issues_csv, '<', 'fix_url_issues.csv') or die "Couldn't open fix_url_issues.csv"; |
|
22 |
my %name2iid = (); |
|
23 |
my %nid2iid = (); |
|
24 |
my %cid2iid = (); |
|
25 |
my %cid2post_nb = (); |
|
26 |
|
|
27 |
while (my $line = <$fix_url_issues_csv>) |
|
28 |
{ |
|
29 |
chomp $line; |
|
30 |
my ($nid, $name, $iid) = split ",", $line; |
|
31 |
$name2iid{$name} = $iid; |
|
32 |
$nid2iid{$nid} = $iid; |
|
33 |
} |
|
34 |
close $fix_url_issues_csv; |
|
35 |
|
|
36 |
open(my $fix_url_comments_csv, '<', 'fix_url_comments.csv') or die "Couldn't open fix_url_comments.csv"; |
|
37 |
while (my $line = <$fix_url_comments_csv>) |
|
38 |
{ |
|
39 |
chomp $line; |
|
40 |
my ($cid, $iid, $post_nb) = split ",", $line; |
|
41 |
$cid2iid{$cid} = $iid; |
|
42 |
$cid2post_nb{$cid} = $post_nb; |
|
43 |
} |
|
44 |
close $fix_url_comments_csv; |
|
45 |
|
|
46 |
|
|
47 |
|
|
48 |
###### Definition of functions |
|
49 |
sub fix_issues_link { |
|
50 |
my $text = $_[0]; |
|
51 |
|
|
52 |
# Find all links in text |
|
53 |
my @links = ($text =~ m/:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content\/t%C3%A2che|node)\/(.*)/g); |
|
54 |
|
|
55 |
if ( @links ) |
|
56 |
{ |
|
57 |
foreach (@links) |
|
58 |
{ |
|
59 |
print $_ ."\n"; |
|
60 |
$text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/.*":http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/$_/#$cid2iid{$_}#note-$cid2post_nb{$_}/g; |
|
61 |
} |
|
62 |
} |
|
63 |
return $text; |
|
64 |
} |
|
65 |
|
|
66 |
sub fix_comments_link { |
|
67 |
my $text = $_[0]; |
|
68 |
|
|
69 |
# Find all links in text |
|
70 |
my @links = $text =~ m/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/.*"/g; |
|
71 |
|
|
72 |
# Foreach link, get comment cid. |
|
73 |
my @cids = (); |
|
74 |
if (@links) |
|
75 |
{ |
|
76 |
foreach (@links) |
|
77 |
{ |
|
78 |
my $cid =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/(.*)#.*"/$1/g; |
|
79 |
push @cids, $cid; |
|
80 |
} |
|
81 |
|
|
82 |
foreach (@cids) |
|
83 |
{ |
|
84 |
$text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/$_#$_":http:\/\/assos\.centrale-marseille\.fr\/comment\/$_#$_/#$name2iid{$_}/g; |
|
85 |
} |
|
86 |
} |
|
87 |
|
|
88 |
return $text; |
|
89 |
} |
|
90 |
|
|
91 |
|
|
92 |
|
|
93 |
###### Update issues |
|
94 |
my $sql_update_issue =<<"SQL"; |
|
95 |
UPDATE issues |
|
96 |
SET author_id = ?, created_on = ? |
|
97 |
WHERE id = ? |
|
98 |
SQL |
|
99 |
|
|
100 |
my $sql_fix_links_issue =<<"SQL"; |
|
101 |
UPDATE issues |
|
102 |
SET description = ? |
|
103 |
WHERE id = ? |
|
104 |
SQL |
|
105 |
|
|
106 |
my $req_update_issue = $dbh->prepare($sql_update_issue); |
|
107 |
my $req_fix_links_issue = $dbh->prepare($sql_fix_links_issue); |
|
108 |
|
|
109 |
# Reading file |
|
110 |
open(my $issues, '<issues.csv') or die "Couldn't open issues.txt\n"; |
|
111 |
|
|
112 |
while (my $line = <$issues>) |
|
113 |
{ |
|
114 |
# Update author and date of creation |
|
115 |
chomp $line; |
|
116 |
|
|
117 |
my ($iid, $author_id, $created_on, $nid, $name) = split ",", $line; |
|
118 |
|
|
119 |
$req_update_issue->execute($author_id, $created_on, $iid); |
|
120 |
|
|
121 |
## Fix links |
|
122 |
# issues |
|
123 |
my $select_description = "SELECT description FROM issues WHERE id = 1569"; |
|
124 |
|
|
125 |
my ($description) = $dbh->selectrow_array($select_description); |
|
126 |
|
|
127 |
$description = fix_issues_link($description); |
|
128 |
|
|
129 |
die "stop"; |
|
130 |
|
|
131 |
print $description . "\n"; |
|
132 |
# print $links[0]; |
|
133 |
# print $name2iid{$links[0]}; |
|
134 |
# Comments |
|
135 |
# $description = fix_comments_link($description); |
|
136 |
|
|
137 |
# $req_fix_links_issue->execute($description, $iid); |
|
138 |
} |
|
139 |
|
|
140 |
$req_update_issue->finish; |
|
141 |
$req_fix_links_issue->finish; |
|
142 |
|
|
143 |
|
|
144 |
###### Update journals |
|
145 |
my $sql_get_id =<<"SQL"; |
|
146 |
SELECT id FROM journals WHERE journalized_id = ? |
|
147 |
ORDER BY created_on ASC |
|
148 |
SQL |
|
149 |
|
|
150 |
my $sql_update_journals =<<"SQL"; |
|
151 |
UPDATE journals |
|
152 |
SET created_on = ?, user_id = ? |
|
153 |
WHERE id = ? |
|
154 |
SQL |
|
155 |
|
|
156 |
my $sql_updated_on =<<"SQL"; |
|
157 |
UPDATE issues SET updated_on = ? |
|
158 |
WHERE id = ? |
|
159 |
SQL |
|
160 |
|
|
161 |
my $sql_fix_links_comment =<<"SQL"; |
|
162 |
UPDATE journals |
|
163 |
SET notes = ? |
|
164 |
WHERE id = ? |
|
165 |
SQL |
|
166 |
|
|
167 |
my $req_get_id = $dbh->prepare($sql_get_id); |
|
168 |
my $req_update_journals = $dbh->prepare($sql_update_journals); |
|
169 |
my $req_updated_on = $dbh->prepare($sql_updated_on); |
|
170 |
my $req_fix_links_comment = $dbh->prepare($sql_fix_links_comment); |
|
171 |
|
|
172 |
# Reading file. |
|
173 |
open(my $comments, '<comments.csv') or die "Couldn't open comments.txt\n"; |
|
174 |
|
|
175 |
# We must remember current iid in order to fetch result or execute query: req_get_id return |
|
176 |
# more than one value |
|
177 |
my $current_iid = -1; |
|
178 |
|
|
179 |
while (my $line = <$comments>) |
|
180 |
{ |
|
181 |
chomp $line; |
|
182 |
|
|
183 |
my ($iid, $user_id, $created_on) = split ",", $line; |
|
184 |
|
|
185 |
# We get the id of the comment. |
|
186 |
if ($current_iid != $iid) |
|
187 |
{ |
|
188 |
$current_iid = $iid; |
|
189 |
$req_get_id->execute($iid); |
|
190 |
} |
|
191 |
|
|
192 |
my ($id) = $req_get_id->fetchrow_array; |
|
193 |
|
|
194 |
# We do the update. |
|
195 |
$req_update_journals->execute($created_on, $user_id, $id); |
|
196 |
$req_updated_on->execute($created_on, $iid); |
|
197 |
|
|
198 |
## Fix links. |
|
199 |
# issues |
|
200 |
my $select_notes = "SELECT notes FROM journals WHERE id = $id"; |
|
201 |
my ($notes) = $dbh->selectrow_array($select_notes); |
|
202 |
# $notes = fix_issues_link($notes); |
|
203 |
# Comments |
|
204 |
# $notes = fix_comments_link($notes); |
|
205 |
|
|
206 |
# $req_fix_links_comment->execute($notes, $id); |
|
207 |
} |
|
208 |
|
|
209 |
$req_get_id->finish; |
|
210 |
$req_update_journals->finish; |
|
211 |
$req_updated_on->finish; |
|
212 |
|
|
213 |
####### Close the db connection. |
|
214 |
$dbh->disconnect; |
scripts_divers/migrer_taches_vers_redmine/fix-db.rb | ||
---|---|---|
1 |
#!/usr/bin/ruby -w |
|
2 |
#encoding: UTF-8 |
|
3 |
|
|
4 |
require "dbi" |
|
5 |
require "uri" |
|
6 |
|
|
7 |
###### Definition of functions |
|
8 |
def custom_chomp(string) |
|
9 |
# Chomp line completely |
|
10 |
previous_string = '' |
|
11 |
while string != previous_string |
|
12 |
previous_string = string.dup # = makes a reference to the original string |
|
13 |
string.chomp! |
|
14 |
end |
|
15 |
return string |
|
16 |
end |
|
17 |
|
|
18 |
|
|
19 |
def fix_issues_link(text, name2iid) |
|
20 |
# Find all links in text |
|
21 |
scan_regex = /:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/([^ \n)]*)/ |
|
22 |
|
|
23 |
# Non flattened output is [['some'], ['text']] |
|
24 |
# flattened output is ['some', 'text'] |
|
25 |
names = text.scan(scan_regex).flatten |
|
26 |
|
|
27 |
if not names.empty? |
|
28 |
names.each do |name| |
|
29 |
name = custom_chomp(name) |
|
30 |
unescaped_name = URI.unescape(name) |
|
31 |
gsub_regex = /"http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/[^ ]*":http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/#{name}/ |
|
32 |
remplace_regex = "#{unescaped_name} : ##{name2iid[name]}" |
|
33 |
text.gsub!(gsub_regex, remplace_regex) |
|
34 |
end |
|
35 |
end |
|
36 |
return text |
|
37 |
end |
|
38 |
|
|
39 |
|
|
40 |
def fix_comments_link(text, cid2iid, cid2post_nb) |
|
41 |
scan_regex = /:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/|\/portail\/)comment\/(\d+)/ |
|
42 |
|
|
43 |
cids = text.scan(scan_regex).flatten |
|
44 |
|
|
45 |
if not cids.empty? |
|
46 |
cids.each do |cid| |
|
47 |
cid = custom_chomp(cid) |
|
48 |
gsub_regex = /"http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)comment\/.*":http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)comment\/#{cid}#comment-\d+/ |
|
49 |
post_nb = custom_chomp(cid2post_nb[cid]) |
|
50 |
remplace_regex = "##{cid2iid[cid]}#note-#{post_nb}" |
|
51 |
text.gsub!(gsub_regex, remplace_regex) |
|
52 |
end |
|
53 |
end |
|
54 |
return text |
|
55 |
end |
|
56 |
|
|
57 |
|
|
58 |
|
|
59 |
###### DATABASE |
|
60 |
begin |
|
61 |
##### Create DB connection |
|
62 |
db = 'redmine_default' |
|
63 |
dbuser = 'root' |
|
64 |
dbpwd = 'tata' |
|
65 |
host = 'localhost' |
|
66 |
dbh = DBI.connect("DBI:Mysql:#{db}:#{host}", dbuser, dbpwd) |
|
67 |
dbh.do("SET NAMES 'UTF8'") |
|
68 |
|
|
69 |
###### Initialisation of hashes needed to fix URL |
|
70 |
name2iid = Hash.new |
|
71 |
nid2iid = Hash.new |
|
72 |
cid2iid = Hash.new |
|
73 |
cid2post_nb = Hash.new |
|
74 |
|
|
75 |
fix_url_issues_csv = File.open("fix_url_issues.csv", "r") |
|
76 |
fix_url_issues_csv.each do |line| |
|
77 |
line = line.chomp |
|
78 |
nid, name, iid = line.split(',') |
|
79 |
name2iid[name] = nid |
|
80 |
nid2iid[nid] = iid |
|
81 |
end |
|
82 |
fix_url_issues_csv.close |
|
83 |
|
|
84 |
fix_url_comments_csv = File.open("fix_url_comments.csv", "r") |
|
85 |
fix_url_comments_csv.each do |line| |
|
86 |
cid, iid, post_nb = line.split(',') |
|
87 |
cid2iid[cid] = iid |
|
88 |
cid2post_nb[cid] = post_nb |
|
89 |
end |
|
90 |
fix_url_comments_csv.close |
|
91 |
|
|
92 |
|
|
93 |
|
|
94 |
###### Update issues |
|
95 |
sql_update_issue = "UPDATE issues SET author_id = ?, created_on = ?, is_private = 1 WHERE id = ?" |
|
96 |
sql_fix_links_issue = "UPDATE issues SET description = ? WHERE id = ?" |
|
97 |
select_description = "SELECT description FROM issues WHERE id = ?" |
|
98 |
|
|
99 |
req_update_issue = dbh.prepare(sql_update_issue) |
|
100 |
req_fix_links_issue = dbh.prepare(sql_fix_links_issue) |
|
101 |
req_select_description = dbh.prepare(select_description) |
|
102 |
|
|
103 |
# Reading file |
|
104 |
issues = File.open('issues.csv', 'r') |
|
105 |
issues.each do |line| |
|
106 |
# Update author and date of creation |
|
107 |
line = line.chomp |
|
108 |
iid, author_id, created_on, nid, name = line.split(',') |
|
109 |
req_update_issue.execute(author_id, created_on, iid) |
|
110 |
|
|
111 |
## Fix links |
|
112 |
# issues |
|
113 |
req_select_description.execute(iid) |
|
114 |
|
|
115 |
description = req_select_description.fetch[0] |
|
116 |
description = fix_issues_link(description, name2iid) |
|
117 |
|
|
118 |
# Comments |
|
119 |
description = fix_comments_link(description, cid2iid, cid2post_nb) |
|
120 |
|
|
121 |
# We update |
|
122 |
req_fix_links_issue.execute(description, iid) |
|
123 |
end |
|
124 |
|
|
125 |
req_select_description.finish |
|
126 |
req_update_issue.finish |
|
127 |
req_fix_links_issue.finish |
|
128 |
|
|
129 |
|
|
130 |
|
|
131 |
###### Update journals |
|
132 |
sql_get_ids = "SELECT id FROM journals WHERE journalized_id = ? ORDER BY id ASC" |
|
133 |
sql_update_journals = "UPDATE journals SET created_on = ?, user_id = ? WHERE id = ?" |
|
134 |
sql_updated_on = "UPDATE issues SET updated_on = ? WHERE id = ?" |
|
135 |
sql_fix_links_comment = "UPDATE journals SET notes = ? WHERE id = ?" |
|
136 |
|
|
137 |
req_get_ids = dbh.prepare(sql_get_ids) |
|
138 |
req_update_journals = dbh.prepare(sql_update_journals) |
|
139 |
req_updated_on = dbh.prepare(sql_updated_on) |
|
140 |
req_fix_links_comment = dbh.prepare(sql_fix_links_comment) |
|
141 |
|
|
142 |
# Reading file |
|
143 |
comments = File.open('comments.csv', 'r') |
|
144 |
|
|
145 |
# We must remember current iid in order to fetch result or execute query: req_get_id return |
|
146 |
# more than one value |
|
147 |
current_iid = -1 |
|
148 |
select_notes = "SELECT notes FROM journals WHERE id = ?" |
|
149 |
req_select_notes = dbh.prepare(select_notes) |
|
150 |
|
|
151 |
comments.each do |line| |
|
152 |
line = line.chomp |
|
153 |
iid, user_id, created_on = line.split(',') |
|
154 |
|
|
155 |
# We get the id of the comment |
|
156 |
if iid != current_iid |
|
157 |
current_iid = iid |
|
158 |
req_get_ids.execute(iid) |
|
159 |
end |
|
160 |
|
|
161 |
id = req_get_ids.fetch[0] |
|
162 |
|
|
163 |
# We do the update |
|
164 |
req_update_journals.execute(created_on, user_id, id) |
|
165 |
req_updated_on.execute(created_on, iid) |
|
166 |
|
|
167 |
## Fix links |
|
168 |
# issues |
|
169 |
req_select_notes.execute(id) |
|
170 |
notes = req_select_notes.fetch[0] |
|
171 |
notes = fix_issues_link(notes, name2iid) |
|
172 |
|
|
173 |
# Comments |
|
174 |
notes = fix_comments_link(notes, cid2iid, cid2post_nb) |
|
175 |
|
|
176 |
req_fix_links_comment.execute(notes, id) |
|
177 |
end |
|
178 |
comments.close |
|
179 |
|
|
180 |
req_select_notes.finish |
|
181 |
req_get_ids.finish |
|
182 |
req_update_journals.finish |
|
183 |
req_updated_on.finish |
|
184 |
rescue DBI::DatabaseError => e |
|
185 |
puts "An error occurred" |
|
186 |
puts "Error code: #{e.err}" |
|
187 |
puts "Error message: #{e.errstr}" |
|
188 |
ensure |
|
189 |
# disconnect from server |
|
190 |
dbh.disconnect if dbh |
|
191 |
end |
scripts_divers/migrer_taches_vers_redmine/goto_redmine.py | ||
---|---|---|
38 | 38 |
has_image = False |
39 | 39 |
# In textile, images are between ! |
40 | 40 |
images = REGEXP_FIND_IMG.findall(txt) |
41 |
print(images) |
|
42 | 41 |
if images: |
43 | 42 |
has_image = True |
44 | 43 |
for image in images: |
45 | 44 |
img_name = REGEXP_NAME_IMG.sub(r'!\1!', image) |
46 |
txt.replace(image, img_name) |
|
45 |
txt = txt.replace(image, img_name)
|
|
47 | 46 |
return txt, has_image |
48 | 47 |
|
49 | 48 |
def html2textile(txt): |
scripts_divers/rm-prefix.pl | ||
---|---|---|
1 |
### Ce script a été écrit afin de supprimer les préfixes des tables des bases données. |
|
2 |
### Ces préfixes étaient indispensables quand tous les sites étaient dans une même base de données. |
|
3 |
|
|
4 |
use strict; |
|
5 |
use warnings; |
|
6 |
use DBI; |
|
7 |
|
|
8 |
my $bd = $ARGV[0]; |
|
9 |
my $serveur = 'localhost'; # Il est possible de mettre une adresse IP |
|
10 |
my $identifiant = 'root'; # identifiant |
|
11 |
my $motdepasse = 'tata'; |
|
12 |
my $port = ''; |
|
13 |
|
|
14 |
my ($prefix) = $ARGV[1]; |
|
15 |
|
|
16 |
my $dbh = DBI->connect( "DBI:mysql:database=$bd;host=$serveur;port=$port", $identifiant, $motdepasse, { RaiseError => 1, } ) or die "Connection impossible à la base de données $bd !\n $! \n $@\n$DBI::errstr"; |
|
17 |
|
|
18 |
my @table_names = $dbh->tables; |
|
19 |
foreach my $table(@table_names) |
|
20 |
{ |
|
21 |
#print $table =~ m/$prefix/; |
|
22 |
#print $table . "\n"; |
|
23 |
|
|
24 |
if( $table =~ m/^`$bd`\.`$prefix/ ) |
|
25 |
{ |
|
26 |
my ($new_name) = ( $table =~ m/$prefix(.*)`/ ); |
|
27 |
my $new_table = "`$bd`.`$new_name`"; |
|
28 |
print $new_name . "\n"; |
|
29 |
print $new_table . "\n"; |
|
30 |
$dbh->do('DROP TABLE IF EXISTS ' . $new_name); |
|
31 |
$dbh->do('RENAME TABLE ' . $table . ' TO ' . $new_name) or die 'Ne peut exécuter la requête'; |
|
32 |
} |
|
33 |
} |
|
34 |
|
|
35 |
$dbh->disconnect(); |
Formats disponibles : Unified diff
fix-db.rb: création.
Ce script doit finir la migration vers redmine en réparant
les liens, les contributeurs et les dates de contributions
directement dans la base de données.
On ne fait plus de perl.
La version ruby corrige les liens de tâche. Reste les liens de commentaire
et à patcher effectivement la base de données.
rm_prefix.pl: ne sert plus à rien, maintenant qu’on a une version python