Projet

Général

Profil

Révision f5ddb21e

Ajouté par Julien Enselme il y a plus de 10 ans

fix-db.rb: création.

Ce script doit finir la migration vers redmine en réparant
les liens, les contributeurs et les dates de contributions
directement dans la base de données.
On ne fait plus de perl.
La version ruby corrige les liens de tâche. Reste les liens de commentaire
et à patcher effectivement la base de données.

rm_prefix.pl: ne sert plus à rien, maintenant qu’on a une version python

Voir les différences:

scripts_divers/migrer_taches_vers_redmine/README
1
* Migrate
2
Simply use the following commands in a clean environment: the folder in which
3
you launch these commands must not contains any csv or txt files from a previous
4
migration otherwise fix-db.rb won't work.
5
- python3 post_redmine.py
6
- ruby fix-db.rb
7

  
8
* Clean
9
Use in a python3 interpreter
10

  
11
#BEGIN_SRC python
12
import post_redmine
13
redmine = post_redmine.Redmine()
14
redmine.sweep()
15
#END_SRC
16

  
17
* Export as html
18
The export-issues-html.sh script allows you to export issues for a projet as
19
html files. It requires access to the database of redmine and only work for
20
mysql.
21

  
22
export-issues-html.sh PROJECT_ID
scripts_divers/migrer_taches_vers_redmine/export-issues-html.sh
1
#!/bin/bash
2

  
3
mysql -h debian -e "SELECT id FROM issues WHERE project_id = $1" -u root -ptata redmine_default > issues_id_list
4

  
5
mkdir issues_html
6
cd issues_html
7
while read line; do
8
    wget http://debian/redmine/issues/$line > /dev/null
9
done < ../issues_id_list
scripts_divers/migrer_taches_vers_redmine/fix-db.pl
1
# In order to complete the migration to Redmine, we need to preserve date
2
# and username. To do this, we need to fix the database.
3

  
4
use strict;
5
use warnings;
6
use DBI;
7

  
8
my $db		= 'redmine_default';
9
my $server	= 'localhost'; # Il est possible de mettre une adresse IP
10
my $user        = 'root';      # identifiant
11
my $passwd	= 'tata';
12
my $port	= '';
13

  
14

  
15
##### Create DB connection
16
my $dbh = DBI->connect( "DBI:mysql:database=$db;host=$server;port=$port", $user, $passwd, { RaiseError => 1, } ) or die "Connection impossible à la base de données $db !\n $! \n $@\n$DBI::errstr";
17

  
18

  
19

  
20
###### Initialisation of hashes needed to fix URL
21
open(my $fix_url_issues_csv, '<', 'fix_url_issues.csv') or die "Couldn't open fix_url_issues.csv";
22
my %name2iid = ();
23
my %nid2iid = ();
24
my %cid2iid = ();
25
my %cid2post_nb = ();
26

  
27
while (my $line = <$fix_url_issues_csv>)
28
  {
29
    chomp $line;
30
    my ($nid, $name, $iid) = split ",", $line;
31
    $name2iid{$name} = $iid;
32
    $nid2iid{$nid} = $iid;
33
  }
34
close $fix_url_issues_csv;
35

  
36
open(my $fix_url_comments_csv, '<', 'fix_url_comments.csv') or die "Couldn't open fix_url_comments.csv";
37
while (my $line = <$fix_url_comments_csv>)
38
  {
39
    chomp $line;
40
    my ($cid, $iid, $post_nb) = split ",", $line;
41
    $cid2iid{$cid} = $iid;
42
    $cid2post_nb{$cid} = $post_nb;
43
  }
44
close $fix_url_comments_csv;
45

  
46

  
47

  
48
###### Definition of functions
49
sub fix_issues_link {
50
    my $text = $_[0];
51

  
52
    # Find all links in text
53
    my @links = ($text =~ m/:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content\/t%C3%A2che|node)\/(.*)/g);
54

  
55
    if ( @links )
56
    {
57
	foreach (@links)
58
	{
59
	    print $_ ."\n";
60
	    $text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/.*":http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)(content\/t%C3%A2che|node)\/$_/#$cid2iid{$_}#note-$cid2post_nb{$_}/g;
61
	}
62
    }
63
    return $text;
64
}
65

  
66
sub fix_comments_link {
67
    my $text = $_[0];
68

  
69
    # Find all links in text
70
    my @links = $text =~ m/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/.*"/g;
71

  
72
     # Foreach link, get comment cid.
73
    my @cids = ();
74
    if (@links)
75
    {
76
	foreach (@links)
77
	{
78
	    my $cid =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/(.*)#.*"/$1/g;
79
	    push @cids, $cid;
80
	}
81

  
82
	foreach (@cids)
83
	{
84
	    $text =~ s/"http:\/\/assos\.centrale-marseille\.fr(\/|\/lessive\/)comment\/$_#$_":http:\/\/assos\.centrale-marseille\.fr\/comment\/$_#$_/#$name2iid{$_}/g;
85
	}
86
    }
87

  
88
    return $text;
89
}
90

  
91

  
92

  
93
###### Update issues
94
my $sql_update_issue =<<"SQL";
95
UPDATE issues
96
  SET author_id = ?, created_on = ?
97
  WHERE  id = ?
98
SQL
99

  
100
my $sql_fix_links_issue =<<"SQL";
101
UPDATE issues
102
    SET description = ?
103
    WHERE id = ?
104
SQL
105

  
106
my $req_update_issue = $dbh->prepare($sql_update_issue);
107
my $req_fix_links_issue = $dbh->prepare($sql_fix_links_issue);
108

  
109
# Reading file
110
open(my $issues, '<issues.csv') or die "Couldn't open issues.txt\n";
111

  
112
while (my $line = <$issues>)
113
  {
114
    # Update author and date of creation
115
    chomp $line;
116

  
117
    my ($iid, $author_id, $created_on, $nid, $name) = split ",", $line;
118

  
119
    $req_update_issue->execute($author_id, $created_on, $iid);
120

  
121
    ## Fix links
122
    # issues
123
    my $select_description = "SELECT description FROM issues WHERE id = 1569";
124

  
125
    my ($description) = $dbh->selectrow_array($select_description);
126

  
127
    $description = fix_issues_link($description);
128

  
129
    die "stop";
130

  
131
    print $description . "\n";
132
#    print $links[0];
133
#    print $name2iid{$links[0]};
134
    # Comments
135
#    $description = fix_comments_link($description);
136

  
137
#    $req_fix_links_issue->execute($description, $iid);
138
  }
139

  
140
$req_update_issue->finish;
141
$req_fix_links_issue->finish;
142

  
143

  
144
###### Update journals
145
my $sql_get_id =<<"SQL";
146
SELECT id FROM journals WHERE journalized_id = ?
147
ORDER BY created_on ASC
148
SQL
149

  
150
my $sql_update_journals =<<"SQL";
151
UPDATE journals
152
SET created_on = ?, user_id = ?
153
WHERE id = ?
154
SQL
155

  
156
my $sql_updated_on =<<"SQL";
157
UPDATE issues SET updated_on = ?
158
WHERE id = ?
159
SQL
160

  
161
my $sql_fix_links_comment =<<"SQL";
162
UPDATE journals
163
    SET notes = ?
164
    WHERE id = ?
165
SQL
166

  
167
my $req_get_id = $dbh->prepare($sql_get_id);
168
my $req_update_journals = $dbh->prepare($sql_update_journals);
169
my $req_updated_on = $dbh->prepare($sql_updated_on);
170
my $req_fix_links_comment = $dbh->prepare($sql_fix_links_comment);
171

  
172
# Reading file.
173
open(my $comments, '<comments.csv') or die "Couldn't open comments.txt\n";
174

  
175
# We must remember current iid in order to fetch result or execute query: req_get_id return
176
# more than one value
177
my $current_iid = -1;
178

  
179
while (my $line = <$comments>)
180
  {
181
    chomp $line;
182

  
183
    my ($iid, $user_id, $created_on) = split ",", $line;
184

  
185
    # We get the id of the comment.
186
    if ($current_iid != $iid)
187
    {
188
	$current_iid = $iid;
189
	$req_get_id->execute($iid);
190
    }
191

  
192
    my ($id) = $req_get_id->fetchrow_array;
193

  
194
    # We do the update.
195
    $req_update_journals->execute($created_on, $user_id, $id);
196
    $req_updated_on->execute($created_on, $iid);
197

  
198
    ## Fix links.
199
    # issues
200
    my $select_notes = "SELECT notes FROM journals WHERE id = $id";
201
    my ($notes) = $dbh->selectrow_array($select_notes);
202
#    $notes = fix_issues_link($notes);
203
    # Comments
204
#    $notes = fix_comments_link($notes);
205

  
206
#    $req_fix_links_comment->execute($notes, $id);
207
  }
208

  
209
$req_get_id->finish;
210
$req_update_journals->finish;
211
$req_updated_on->finish;
212

  
213
####### Close the db connection.
214
$dbh->disconnect;
scripts_divers/migrer_taches_vers_redmine/fix-db.rb
1
#!/usr/bin/ruby -w
2
#encoding: UTF-8
3

  
4
require "dbi"
5
require "uri"
6

  
7
###### Definition of functions
8
def custom_chomp(string)
9
  # Chomp line completely
10
  previous_string = ''
11
  while string != previous_string
12
    previous_string = string.dup # = makes a reference to the original string
13
    string.chomp!
14
  end
15
  return string
16
end
17

  
18

  
19
def fix_issues_link(text, name2iid)
20
  # Find all links in text
21
  scan_regex =  /:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/([^ \n)]*)/
22

  
23
  # Non flattened output is [['some'], ['text']]
24
  # flattened output is ['some', 'text']
25
  names = text.scan(scan_regex).flatten
26

  
27
  if not names.empty?
28
    names.each do |name|
29
      name = custom_chomp(name)
30
      unescaped_name = URI.unescape(name)
31
      gsub_regex = /"http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/[^ ]*":http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)(?:content(?:\/t%C3%A2che)?|node)\/#{name}/
32
      remplace_regex = "#{unescaped_name} : ##{name2iid[name]}"
33
      text.gsub!(gsub_regex, remplace_regex)
34
    end
35
  end
36
  return text
37
end
38

  
39

  
40
def fix_comments_link(text, cid2iid, cid2post_nb)
41
  scan_regex = /:http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/|\/portail\/)comment\/(\d+)/
42

  
43
  cids = text.scan(scan_regex).flatten
44

  
45
  if not cids.empty?
46
    cids.each do |cid|
47
      cid = custom_chomp(cid)
48
      gsub_regex = /"http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)comment\/.*":http:\/\/assos\.centrale-marseille\.fr(?:\/|\/lessive\/)comment\/#{cid}#comment-\d+/
49
      post_nb = custom_chomp(cid2post_nb[cid])
50
      remplace_regex = "##{cid2iid[cid]}#note-#{post_nb}"
51
      text.gsub!(gsub_regex, remplace_regex)
52
    end
53
  end
54
  return text
55
end
56

  
57

  
58

  
59
###### DATABASE
60
begin
61
  ##### Create DB connection
62
  db = 'redmine_default'
63
  dbuser = 'root'
64
  dbpwd = 'tata'
65
  host = 'localhost'
66
  dbh = DBI.connect("DBI:Mysql:#{db}:#{host}", dbuser, dbpwd)
67
  dbh.do("SET NAMES 'UTF8'")
68

  
69
  ###### Initialisation of hashes needed to fix URL
70
  name2iid = Hash.new
71
  nid2iid = Hash.new
72
  cid2iid = Hash.new
73
  cid2post_nb = Hash.new
74

  
75
  fix_url_issues_csv = File.open("fix_url_issues.csv", "r")
76
  fix_url_issues_csv.each do |line|
77
    line = line.chomp
78
    nid, name, iid = line.split(',')
79
    name2iid[name] = nid
80
    nid2iid[nid] = iid
81
  end
82
  fix_url_issues_csv.close
83

  
84
  fix_url_comments_csv = File.open("fix_url_comments.csv", "r")
85
  fix_url_comments_csv.each do |line|
86
    cid, iid, post_nb = line.split(',')
87
    cid2iid[cid] = iid
88
    cid2post_nb[cid] = post_nb
89
  end
90
  fix_url_comments_csv.close
91

  
92

  
93

  
94
  ###### Update issues
95
  sql_update_issue = "UPDATE issues SET author_id = ?, created_on = ?, is_private = 1 WHERE  id = ?"
96
  sql_fix_links_issue = "UPDATE issues SET description = ? WHERE id = ?"
97
  select_description = "SELECT description FROM issues WHERE id = ?"
98

  
99
  req_update_issue = dbh.prepare(sql_update_issue)
100
  req_fix_links_issue = dbh.prepare(sql_fix_links_issue)
101
  req_select_description = dbh.prepare(select_description)
102

  
103
  # Reading file
104
  issues = File.open('issues.csv', 'r')
105
  issues.each do |line|
106
    # Update author and date of creation
107
    line = line.chomp
108
    iid, author_id, created_on, nid, name = line.split(',')
109
    req_update_issue.execute(author_id, created_on, iid)
110

  
111
    ## Fix links
112
    # issues
113
    req_select_description.execute(iid)
114

  
115
    description = req_select_description.fetch[0]
116
    description = fix_issues_link(description, name2iid)
117

  
118
    # Comments
119
    description = fix_comments_link(description, cid2iid, cid2post_nb)
120

  
121
    # We update
122
    req_fix_links_issue.execute(description, iid)
123
  end
124

  
125
  req_select_description.finish
126
  req_update_issue.finish
127
  req_fix_links_issue.finish
128

  
129

  
130

  
131
  ###### Update journals
132
  sql_get_ids = "SELECT id FROM journals WHERE journalized_id = ? ORDER BY id ASC"
133
  sql_update_journals = "UPDATE journals SET created_on = ?, user_id = ? WHERE id = ?"
134
  sql_updated_on = "UPDATE issues SET  updated_on = ? WHERE id = ?"
135
  sql_fix_links_comment = "UPDATE journals SET notes = ? WHERE id = ?"
136

  
137
  req_get_ids = dbh.prepare(sql_get_ids)
138
  req_update_journals = dbh.prepare(sql_update_journals)
139
  req_updated_on = dbh.prepare(sql_updated_on)
140
  req_fix_links_comment = dbh.prepare(sql_fix_links_comment)
141

  
142
  # Reading file
143
  comments = File.open('comments.csv', 'r')
144

  
145
  # We must remember current iid in order to fetch result or execute query: req_get_id return
146
  # more than one value
147
  current_iid = -1
148
  select_notes = "SELECT notes FROM journals WHERE id = ?"
149
  req_select_notes = dbh.prepare(select_notes)
150

  
151
  comments.each do |line|
152
    line = line.chomp
153
    iid, user_id, created_on = line.split(',')
154

  
155
    # We get the id of the comment
156
    if iid != current_iid
157
      current_iid = iid
158
      req_get_ids.execute(iid)
159
    end
160

  
161
    id = req_get_ids.fetch[0]
162

  
163
    # We do the update
164
    req_update_journals.execute(created_on, user_id, id)
165
    req_updated_on.execute(created_on, iid)
166

  
167
    ## Fix links
168
    # issues
169
    req_select_notes.execute(id)
170
    notes = req_select_notes.fetch[0]
171
    notes = fix_issues_link(notes, name2iid)
172

  
173
    # Comments
174
    notes = fix_comments_link(notes, cid2iid, cid2post_nb)
175

  
176
    req_fix_links_comment.execute(notes, id)
177
  end
178
  comments.close
179

  
180
  req_select_notes.finish
181
  req_get_ids.finish
182
  req_update_journals.finish
183
  req_updated_on.finish
184
rescue DBI::DatabaseError => e
185
     puts "An error occurred"
186
     puts "Error code:    #{e.err}"
187
     puts "Error message: #{e.errstr}"
188
ensure
189
     # disconnect from server
190
     dbh.disconnect if dbh
191
end
scripts_divers/migrer_taches_vers_redmine/goto_redmine.py
38 38
    has_image = False
39 39
    # In textile, images are between !
40 40
    images = REGEXP_FIND_IMG.findall(txt)
41
    print(images)
42 41
    if images:
43 42
        has_image = True
44 43
        for image in images:
45 44
            img_name = REGEXP_NAME_IMG.sub(r'!\1!', image)
46
            txt.replace(image, img_name)
45
            txt = txt.replace(image, img_name)
47 46
    return txt, has_image
48 47

  
49 48
def html2textile(txt):
scripts_divers/rm-prefix.pl
1
### Ce script a été écrit afin de supprimer les préfixes des tables des bases données.
2
### Ces préfixes étaient indispensables quand tous les sites étaient dans une même base de données.
3

  
4
use strict;
5
use warnings;
6
use DBI;
7

  
8
my $bd		= $ARGV[0];
9
my $serveur	= 'localhost'; # Il est possible de mettre une adresse IP
10
my $identifiant = 'root';      # identifiant
11
my $motdepasse	= 'tata';
12
my $port	= '';
13

  
14
my ($prefix) = $ARGV[1];
15

  
16
my $dbh = DBI->connect( "DBI:mysql:database=$bd;host=$serveur;port=$port", $identifiant, $motdepasse, { RaiseError => 1, } ) or die "Connection impossible à la base de données $bd !\n $! \n $@\n$DBI::errstr";
17

  
18
my @table_names = $dbh->tables;
19
foreach my $table(@table_names)
20
{
21
    #print $table =~ m/$prefix/;
22
    #print $table . "\n";
23

  
24
    if( $table =~ m/^`$bd`\.`$prefix/ )
25
    {
26
	my ($new_name) = ( $table =~ m/$prefix(.*)`/ );
27
	my $new_table = "`$bd`.`$new_name`";
28
	print $new_name . "\n";
29
	print $new_table . "\n";
30
	$dbh->do('DROP TABLE IF EXISTS ' . $new_name);
31
	$dbh->do('RENAME TABLE ' . $table . ' TO ' . $new_name) or die 'Ne peut exécuter la requête';
32
    }
33
}
34

  
35
$dbh->disconnect();

Formats disponibles : Unified diff